jjb/onap/cps/prepare-performance-tests-data.sh

   1 #!/bin/bash
   2 sudo apt-get install bc
   3 #############################################################################################################################
   4 ################################################ F U N C T I O N S ##########################################################
   5 #############################################################################################################################
   6
   7 create_data="create_performance_data.txt"
   8 read_data="read_performance_data.txt"
   9 update_data="update_performance_data.txt"
  10 delete_data="delete_performance_data.txt"
  11
  12 # Data files headers
  13 create_data_title_1="Creating openroadm anchors with large data"
  14 read_data_title_1="Read datatrees using openroadm root"
  15 update_data_title_1="Update 100 data nodes"
  16 delete_data_title_1="Batch delete 100 whole lists"
  17
  18 # Text patterns to match in console log
  19 creating_bookstore_pattern="^.*Creating openroadm anchors with large.*"
  20 read_datatreees_pattern="^.*Read datatrees using openroadm root.*"
  21 update_datanodes_pattern="^.*Update 100 data nodes.*"
  22 batch_delete_pattern="^.*Batch delete 100 whole lists.*"
  23
  24 new_data=""
  25 JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
  26
  27 # Get latest build
  28 getLatestBuild() {
  29     latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
  30     echo "latest build is $latestBuild"
  31 }
  32
  33 latestBuild=$(getLatestBuild)
  34 latestBuildToRecord=""
  35 consoleText=""
  36 lastRecordedBuild=""
  37 latestRecordedBuild=""
  38 timestampOfLatestRecordedBuild=""
  39
  40 # Create and add header to the file
  41 createAndAddHeader() {
  42     file="$1"
  43     header="$2"
  44     if [[ ! -f "$file" ]]; then
  45         echo "Creating $file"
  46         touch "$file"
  47         echo "Build,$header" > $file
  48     else
  49         echo "$file already exists"
  50     fi
  51 }
  52
  53 # Get the console text from specific build of the performance job
  54 getConsoleText() {
  55     buildToRead=$1
  56     consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
  57     consoleText=$(curl -s "$consoleURL")
  58 }
  59
  60 getAndRecordDataResults() {
  61     consoleText=$1
  62     patternToMatch=$2
  63     dataFile=$3
  64     buildNumber=$4
  65     new_data=""
  66     matched_line=""
  67     limit_value=""
  68     took_value=""
  69
  70     # Get and calculate data for plot
  71     if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then
  72         echo "Matched line: $matched_line"
  73         limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(\.\d+)?' | tr -cd '[:digit:].')
  74         echo $limit_value
  75         took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(\.\d+)?' | tr -cd '[:digit:].')
  76         echo $took_value
  77         percentage=$(echo "scale=2; $took_value * 100.00 / $limit_value" | bc)
  78
  79
  80         echo "Limit value: $limit_value"
  81         echo "Took value: $took_value"
  82         echo "Percentage: $percentage"
  83
  84     if [ -z "$new_data" ]; then
  85         new_data="$percentage"
  86         else
  87         new_data+=",$percentage"
  88         fi
  89         echo "New data: $new_data"
  90
  91         else
  92         echo "No match found."
  93         fi
  94
  95     # Record result
  96     lastLine=$(tail -n 1 "$dataFile")
  97     newLine="$buildNumber,$new_data"
  98     if [ -z "$new_data" ]; then
  99         echo "No new data"
 100         lastRecordedBuild=$(echo "$lastLine" | awk -F, '{print $1}')
 101         recordLatestRecordedBuild "$lastRecordedBuild"
 102     elif [ $newLine == $lastLine ]; then
 103           echo "Data already exists"
 104         recordLatestRecordedBuild "$buildNumber"
 105     else
 106         echo "$buildNumber,$new_data" >> $dataFile
 107         echo "New data added"
 108         recordLatestRecordedBuild "$buildNumber"
 109     fi
 110 }
 111
 112 recordLatestRecordedBuild() {
 113         latestBuildToRecord="$1"
 114   timestampOfLatestRecordedBuild=$(curl -s "${JENKINS_JOB_URL}/${latestBuildToRecord}/api/json" | jq -r '.timestamp')
 115         formattedTimestampOfLatestRecordedBuild=$(date -d "@$((timestampOfLatestRecordedBuild / 1000))" "+%B %e, %Y at %H:%M")
 116     echo "Latest recorded build: $latestBuildToRecord"
 117     echo "Date of latest recorded build: $formattedTimestampOfLatestRecordedBuild"
 118     latestRecordedBuild=$latestBuildToRecord
 119 }
 120
 121 buildStaticReport() {
 122     dataFile="$1"  # Get the input file name from the function parameter
 123     chartTitle="$2"
 124
 125     sudo apt-get update
 126     sudo apt-get install -y gnuplot
 127
 128     # Create a temporary Gnuplot script
 129     cat <<EOT > gnuplot_script.gp
 130     set datafile separator ","
 131     set terminal pngcairo size 1500,600
 132     set output "${chartTitle}.png"
 133
 134     # Set X-axis label
 135     set xlabel "Build"
 136
 137     # Set Y-axis label
 138     set ylabel "Percentage of limit %"
 139
 140           # Get stats for min and max
 141     stats '$dataFile' using 1
 142           xmin = STATS_min
 143           freq = 5 #frequency
 144           xLabel(x) = (int((x - xmin) / freq) % 2 == 0) ? sprintf("%d", x) : ""
 145
 146         plot '$dataFile' using (column(0)+1):2:xtic(xLabel(column(1))) with linespoints title "limit"
 147 EOT
 148
 149     # Run the Gnuplot script
 150     gnuplot gnuplot_script.gp
 151
 152     # Remove the temporary script
 153     rm gnuplot_script.gp
 154 }
 155
 156 buildHTMLReport() {
 157
 158     chartTitle1="$1" # i.e Get the chart file name as the first parameter
 159     chartFileName1="$2"
 160
 161     chartTitle2="$3"
 162     chartFileName2="$4"
 163
 164     chartTitle3="$5"
 165     chartFileName3="$6"
 166
 167     chartTitle4="$7"
 168     chartFileName4="$8"
 169
 170     reportTitle="$9"     # i.e Get the report title as the ninth parameter
 171
 172 cat <<EOT > "index.html"
 173 <!DOCTYPE html>
 174 <html>
 175 <head>
 176   <title>$reportTitle</title>
 177 </head>
 178 <body>
 179     <h1 style="text-align: center;">$reportTitle</h1>
 180     <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
 181     <p>The performance tests job runs daily at 02:15 UTC, providing performance metrics. The following graphs update at 04:15 UTC.</p>
 182     <p>Successful performance tests job build adds new data, but even if a build fails, existing data is retained.</p>
 183     <p>Updates occur whenever new successful data is available.</p>
 184
 185     <table align="center">
 186         <tr> <!-- First Row -->
 187             <td align="center">
 188                 <figcaption>"$chartTitle1"</figcaption>
 189                 <img src="$chartFileName1" alt="Image 1" width="750" height="300">
 190             </td>
 191             <td align="center" style="padding: 10px;">
 192                 <figcaption>"$chartTitle2"</figcaption>
 193                 <img src="$chartFileName2" alt="Image 2" width="750" height="300">
 194             </td>
 195         </tr>
 196         <tr> <!-- Second Row -->
 197             <td align="center" style="padding: 10px;">
 198                 <figcaption>"$chartTitle3"</figcaption>
 199                 <img src="$chartFileName3" alt="Image 3" width="750" height="300">
 200             </td>
 201             <td align="center" style="padding: 10px;">
 202                 <figcaption>"$chartTitle4"</figcaption>
 203                 <img src="$chartFileName4" alt="Image 4" width="750" height="300">
 204             </td>
 205         </tr>
 206     </table>
 207 </body>
 208 </html>
 209 EOT
 210 }
 211
 212 buildPageReport() {
 213     chartFileName="$1"
 214     reportTitle="$2"
 215     outputFile="$3"
 216     cat <<EOT > "$outputFile"
 217     <!DOCTYPE html>
 218     <html>
 219     <head>
 220     <title>$reportTitle</title>
 221     </head>
 222     <body>
 223         <h1>$reportTitle</h1>
 224         <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
 225         <img src="$chartFileName" alt="Graph Image">
 226     </body>
 227     </html>
 228 EOT
 229 }
 230
 231 buildPlotForCreateOperation() {
 232         buildNumber="$1"
 233     createAndAddHeader "$create_data" "$create_data_title_1"
 234     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 235 }
 236
 237 buildPlotForReadOperation() {
 238         buildNumber="$1"
 239     createAndAddHeader "$read_data" "$read_data_title_1"
 240     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 241 }
 242
 243 buildPlotForUpdateOperation() {
 244         buildNumber="$1"
 245     createAndAddHeader "$update_data" "$update_data_title_1"
 246     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 247 }
 248
 249 buildPlotForBatchOperation() {
 250         buildNumber="$1"
 251     createAndAddHeader "$delete_data" "$delete_data_title_1"
 252     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 253 }
 254
 255 #############################################################################################################################
 256 #############################################################################################################################
 257 #############################################################################################################################
 258 #############################################################################################################################
 259
 260 cd $WORKSPACE
 261
 262 getLatestBuild
 263 if [ -z "$(ls -A)" ]; then
 264         # Calculate the starting value for the loop
 265         startValue=$((latestBuild - 100))
 266
 267         # Start the loop from startValue up to latestBuild
 268         for ((i=startValue; i<=latestBuild; i++)); do
 269             getConsoleText "$i"
 270             buildPlotForCreateOperation "$i"
 271             buildPlotForReadOperation "$i"
 272             buildPlotForUpdateOperation "$i"
 273             buildPlotForBatchOperation "$i"
 274         done
 275             buildStaticReport "$create_data" "createLargeData"
 276             buildStaticReport "$read_data" "readDataTrees"
 277             buildStaticReport "$update_data" "updateDatanodes"
 278             buildStaticReport "$delete_data" "batchDelete"
 279 else
 280     getConsoleText $latestBuild
 281     buildPlotForCreateOperation $latestBuild
 282     buildPlotForReadOperation $latestBuild
 283     buildPlotForUpdateOperation $latestBuild
 284     buildPlotForBatchOperation $latestBuild
 285
 286     buildStaticReport "$create_data" "createLargeData"
 287     buildStaticReport "$read_data" "readDataTrees"
 288     buildStaticReport "$update_data" "updateDatanodes"
 289     buildStaticReport "$delete_data" "batchDelete"
 290 fi
 291
 292
 293 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 294
 295 buildHTMLReport \
 296     "Creating openroadm anchors with large data" \
 297     "createLargeData.png" \
 298     "Read datatrees using openroadm root" \
 299     "readDataTrees.png" \
 300     "Update 100 data nodes" \
 301     "updateDatanodes.png" \
 302     "Batch delete 100 whole lists" \
 303     "batchDelete.png" \
 304     "Performance Review"
 305
 306 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 307 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 308 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 309 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
 310
 311
 312 buildPlotForCreateOperation() {
 313         buildNumber="$1"
 314     createAndAddHeader "$create_data" "$create_data_title_1"
 315     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 316 }
 317
 318 buildPlotForReadOperation() {
 319         buildNumber="$1"
 320     createAndAddHeader "$read_data" "$read_data_title_1"
 321     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 322 }
 323
 324 buildPlotForUpdateOperation() {
 325         buildNumber="$1"
 326     createAndAddHeader "$update_data" "$update_data_title_1"
 327     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 328 }
 329
 330 buildPlotForBatchOperation() {
 331         buildNumber="$1"
 332     createAndAddHeader "$delete_data" "$delete_data_title_1"
 333     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 334 }
 335
 336 #############################################################################################################################
 337 #############################################################################################################################
 338 #############################################################################################################################
 339 #############################################################################################################################
 340
 341 cd $WORKSPACE
 342
 343 getLatestBuild
 344 if [ -z "$(ls -A)" ]; then
 345         # Calculate the starting value for the loop
 346         startValue=$((latestBuild - 50))
 347
 348         # Start the loop from startValue up to latestBuild
 349         for ((i=startValue; i<=latestBuild; i++)); do
 350             getConsoleText "$i"
 351             buildPlotForCreateOperation "$i"
 352             buildPlotForReadOperation "$i"
 353             buildPlotForUpdateOperation "$i"
 354             buildPlotForBatchOperation "$i"
 355         done
 356             buildStaticReport "$create_data" "createLargeData"
 357             buildStaticReport "$read_data" "readDataTrees"
 358             buildStaticReport "$update_data" "updateDatanodes"
 359             buildStaticReport "$delete_data" "batchDelete"
 360 else
 361     getConsoleText $latestBuild
 362     buildPlotForCreateOperation $latestBuild
 363     buildPlotForReadOperation $latestBuild
 364     buildPlotForUpdateOperation $latestBuild
 365     buildPlotForBatchOperation $latestBuild
 366
 367     buildStaticReport "$create_data" "createLargeData"
 368     buildStaticReport "$read_data" "readDataTrees"
 369     buildStaticReport "$update_data" "updateDatanodes"
 370     buildStaticReport "$delete_data" "batchDelete"
 371 fi
 372
 373
 374 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 375
 376 buildHTMLReport \
 377     "Creating openroadm anchors with large data" \
 378     "createLargeData.png" \
 379     "Read datatrees using openroadm root" \
 380     "readDataTrees.png" \
 381     "Update 100 data nodes" \
 382     "updateDatanodes.png" \
 383     "Batch delete 100 whole lists" \
 384     "batchDelete.png" \
 385     "Performance Review"
 386
 387 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 388 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 389 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 390 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"