Print summary report for K6 tests
- add a generic template to print summary of k6 test results
- remove container logs since its around 50 mb
Issue-ID: CPS-2215
Change-Id: I8971fc30b9dc8be2ed16eda6755d17a91f608b48
Signed-off-by: halil.cakal <halil.cakal@est.tech>
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Signed-off-by: halil.cakal <halil.cakal@est.tech>
diff --git a/k6-tests/ncmp/1-create-cmhandles.js b/k6-tests/ncmp/1-create-cmhandles.js
index c32cd2f..a6f8086 100644
--- a/k6-tests/ncmp/1-create-cmhandles.js
+++ b/k6-tests/ncmp/1-create-cmhandles.js
@@ -21,36 +21,48 @@
import http from 'k6/http';
import exec from 'k6/execution';
import { check } from 'k6';
-import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds } from './utils.js';
+import {
+ NCMP_BASE_URL,
+ DMI_PLUGIN_URL,
+ TOTAL_CM_HANDLES,
+ makeBatchOfCmHandleIds,
+ makeCustomSummaryReport
+} from './utils.js';
const BATCH_SIZE = 100;
export const options = {
- vus: 1,
- iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 1000'],
- },
+ vus: 1,
+ iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
+ thresholds: {
+ http_req_failed: ['rate == 0'],
+ http_req_duration: ['avg <= 1000'],
+ },
};
-export default function() {
+export default function () {
const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(BATCH_SIZE, exec.scenario.iterationInTest);
const payload = {
"dmiPlugin": DMI_PLUGIN_URL,
"createdCmHandles": nextBatchOfCmHandleIds.map(cmHandleId => ({
"cmHandle": cmHandleId,
- "cmHandleProperties": { "neType": "RadioNode" },
+ "cmHandleProperties": {"neType": "RadioNode"},
"publicCmHandleProperties": {
"Color": "yellow",
"Size": "small",
"Shape": "cube"
- }
+ }
})),
};
const response = http.post(NCMP_BASE_URL + '/ncmpInventory/v1/ch', JSON.stringify(payload), {
- headers: { 'Content-Type': 'application/json' },
+ headers: {'Content-Type': 'application/json'},
});
check(response, {
'status equals 200': (r) => r.status === 200,
});
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/10-mixed-load-test.js
index d13afa5..b2c20e1 100644
--- a/k6-tests/ncmp/10-mixed-load-test.js
+++ b/k6-tests/ncmp/10-mixed-load-test.js
@@ -20,37 +20,39 @@
import http from 'k6/http';
import { check } from 'k6';
-import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js'
+import { NCMP_BASE_URL, getRandomCmHandleId, makeCustomSummaryReport } from './utils.js'
import { searchRequest } from './search-base.js';
export const options = {
- scenarios: {
- passthrough_read: {
- executor: 'constant-vus',
- exec: 'passthrough_read',
- vus: 10,
- duration: '1m',
+ scenarios: {
+ passthrough_read: {
+ executor: 'constant-vus',
+ exec: 'passthrough_read',
+ vus: 10,
+ duration: '1m',
+ },
+ id_search_module: {
+ executor: 'constant-vus',
+ exec: 'id_search_module',
+ vus: 5,
+ duration: '1m',
+ },
+ cm_search_module: {
+ executor: 'constant-vus',
+ exec: 'cm_search_module',
+ vus: 5,
+ duration: '1m',
+ },
},
- id_search_module: {
- executor: 'constant-vus',
- exec: 'id_search_module',
- vus: 5,
- duration: '1m',
- },
- cm_search_module: {
- executor: 'constant-vus',
- exec: 'cm_search_module',
- vus: 5,
- duration: '1m',
- },
- },
- thresholds: {
- http_req_failed: ['rate==0'],
- 'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms
- 'http_req_duration{scenario:id_search_module}': ['avg <= 200'],
- 'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'],
- },
+ thresholds: {
+ 'http_req_failed{scenario:passthrough_read}': ['rate == 0'],
+ 'http_req_failed{scenario:id_search_module}': ['rate == 0'],
+ 'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
+ 'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms
+ 'http_req_duration{scenario:id_search_module}': ['avg <= 200'],
+ 'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'],
+ },
};
export function passthrough_read() {
@@ -68,7 +70,7 @@
"cmHandleQueryParameters": [
{
"conditionName": "hasAllModules",
- "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+ "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
}
]
};
@@ -80,9 +82,15 @@
"cmHandleQueryParameters": [
{
"conditionName": "hasAllModules",
- "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+ "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
}
]
};
searchRequest('searches', JSON.stringify(search_filter));
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
diff --git a/k6-tests/ncmp/11-delete-cmhandles.js b/k6-tests/ncmp/11-delete-cmhandles.js
index daced8d..dabf12c 100644
--- a/k6-tests/ncmp/11-delete-cmhandles.js
+++ b/k6-tests/ncmp/11-delete-cmhandles.js
@@ -21,28 +21,34 @@
import http from 'k6/http';
import exec from 'k6/execution';
import { check } from 'k6';
-import { NCMP_BASE_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds } from './utils.js';
+import { NCMP_BASE_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './utils.js';
const BATCH_SIZE = 100;
export const options = {
- vus: 1,
- iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 1000'],
- },
+ vus: 1,
+ iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
+ thresholds: {
+ http_req_failed: ['rate == 0'],
+ http_req_duration: ['avg <= 1000'],
+ },
};
-export default function() {
+export default function () {
const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(BATCH_SIZE, exec.scenario.iterationInTest);
const payload = {
"dmiPlugin": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092",
"removedCmHandles": nextBatchOfCmHandleIds,
};
const response = http.post(NCMP_BASE_URL + '/ncmpInventory/v1/ch', JSON.stringify(payload), {
- headers: { 'Content-Type': 'application/json' },
+ headers: {'Content-Type': 'application/json'},
});
check(response, {
'status equals 200': (r) => r.status === 200,
});
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
index 04a5c21..4ecadde 100644
--- a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
+++ b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
@@ -20,18 +20,18 @@
import http from 'k6/http';
import { sleep, fail } from 'k6';
-import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
+import { makeCustomSummaryReport, NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
export const options = {
- vus: 1,
- iterations: 1,
- thresholds: {
- http_req_failed: ['rate == 0'],
- iteration_duration: ['max <= 300_000'], // 5 minutes
- },
+ vus: 1,
+ iterations: 1,
+ thresholds: {
+ http_req_failed: ['rate == 0'],
+ iteration_duration: ['max <= 300_000'], // 5 minutes
+ },
};
-export default function() {
+export default function () {
waitForCmHandlesToBeReady(TOTAL_CM_HANDLES);
}
@@ -62,3 +62,9 @@
const cmHandles = jsonData[0]["dmi-reg:dmi-registry"]["cm-handles"];
return cmHandles.filter(cmhandle => cmhandle['state']['cm-handle-state'] === 'READY').length;
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/3-passthrough-read.js b/k6-tests/ncmp/3-passthrough-read.js
index 912cee5..0b8c58a 100644
--- a/k6-tests/ncmp/3-passthrough-read.js
+++ b/k6-tests/ncmp/3-passthrough-read.js
@@ -21,21 +21,21 @@
import http from 'k6/http';
import { check } from 'k6';
import { Trend } from "k6/metrics";
-import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js'
+import { NCMP_BASE_URL, getRandomCmHandleId, makeCustomSummaryReport } from './utils.js'
let ncmpOverheadTrend = new Trend("ncmp_overhead");
export const options = {
- vus: 12,
- duration: '30s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- ncmp_overhead: ['avg <= 40'],
- },
+ vus: 12,
+ duration: '30s',
+ thresholds: {
+ http_req_failed: ['rate == 0'],
+ ncmp_overhead: ['avg <= 40'],
+ },
};
// The function that defines VU logic.
-export default function() {
+export default function () {
const cmHandleId = getRandomCmHandleId();
const datastoreName = 'ncmp-datastore%3Apassthrough-operational';
const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${cmHandleId}/data/ds/${datastoreName}?resourceIdentifier=x&include-descendants=true`
@@ -49,3 +49,9 @@
const overhead = response.timings.duration - dmiDelay;
ncmpOverheadTrend.add(overhead);
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/4-id-search-no-filter.js b/k6-tests/ncmp/4-id-search-no-filter.js
index 774c609..a99f8b5 100644
--- a/k6-tests/ncmp/4-id-search-no-filter.js
+++ b/k6-tests/ncmp/4-id-search-no-filter.js
@@ -19,6 +19,7 @@
*/
import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
export const options = {
vus: 5,
@@ -30,6 +31,12 @@
};
// The function that defines VU logic.
-export default function() {
+export default function () {
searchRequest('id-searches', '{}')
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/5-search-no-filter.js b/k6-tests/ncmp/5-search-no-filter.js
index fa08d0c..f6a7c99 100644
--- a/k6-tests/ncmp/5-search-no-filter.js
+++ b/k6-tests/ncmp/5-search-no-filter.js
@@ -19,6 +19,7 @@
*/
import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
export const options = {
vus: 5,
@@ -29,6 +30,12 @@
},
};
-export default function() {
+export default function () {
searchRequest('searches', '{}')
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/6-id-search-public-property.js b/k6-tests/ncmp/6-id-search-public-property.js
index e8b584a..0a06331 100644
--- a/k6-tests/ncmp/6-id-search-public-property.js
+++ b/k6-tests/ncmp/6-id-search-public-property.js
@@ -19,6 +19,7 @@
*/
import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
export const options = {
vus: 5,
@@ -29,14 +30,20 @@
},
};
-export default function() {
+export default function () {
const search_filter = {
"cmHandleQueryParameters": [
{
"conditionName": "hasAllProperties",
- "conditionParameters": [ {"Color": "yellow"}, {"Size": "small"} ]
+ "conditionParameters": [{"Color": "yellow"}, {"Size": "small"}]
}
]
};
searchRequest('id-searches', JSON.stringify(search_filter));
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/7-search-public-property.js b/k6-tests/ncmp/7-search-public-property.js
index 3e052da..9a8e339 100644
--- a/k6-tests/ncmp/7-search-public-property.js
+++ b/k6-tests/ncmp/7-search-public-property.js
@@ -19,6 +19,7 @@
*/
import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
export const options = {
vus: 5,
@@ -29,14 +30,20 @@
},
};
-export default function() {
+export default function () {
const search_filter = {
"cmHandleQueryParameters": [
{
"conditionName": "hasAllProperties",
- "conditionParameters": [ {"Color": "yellow"}, {"Size": "small"} ]
+ "conditionParameters": [{"Color": "yellow"}, {"Size": "small"}]
}
]
};
searchRequest('searches', JSON.stringify(search_filter));
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/8-id-search-module.js b/k6-tests/ncmp/8-id-search-module.js
index e99fbfe..299fe62 100644
--- a/k6-tests/ncmp/8-id-search-module.js
+++ b/k6-tests/ncmp/8-id-search-module.js
@@ -19,6 +19,7 @@
*/
import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
export const options = {
vus: 5,
@@ -29,14 +30,20 @@
},
};
-export default function() {
+export default function () {
const search_filter = {
"cmHandleQueryParameters": [
{
"conditionName": "hasAllModules",
- "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+ "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
}
]
};
searchRequest('id-searches', JSON.stringify(search_filter));
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/9-search-module.js b/k6-tests/ncmp/9-search-module.js
index 03eb9a6..d340bf2 100644
--- a/k6-tests/ncmp/9-search-module.js
+++ b/k6-tests/ncmp/9-search-module.js
@@ -19,6 +19,7 @@
*/
import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
export const options = {
vus: 5,
@@ -29,14 +30,20 @@
},
};
-export default function() {
+export default function () {
const search_filter = {
"cmHandleQueryParameters": [
{
"conditionName": "hasAllModules",
- "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+ "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
}
]
};
searchRequest('searches', JSON.stringify(search_filter));
}
+
+export function handleSummary(data) {
+ return {
+ stdout: makeCustomSummaryReport(data, options),
+ };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index f75cf2f..80231b9 100755
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -31,13 +31,19 @@
pushd "$(dirname "$0")" || exit 1
+printf "Test Case\tCondition\tLimit\tActual\tResult\n" > summary.log
+
number_of_failures=0
for test_script in "${ALL_TEST_SCRIPTS[@]}"; do
echo "k6 run $test_script"
- k6 --quiet run "$test_script" || ((number_of_failures++))
- echo
+ k6 --quiet run -e K6_MODULE_NAME="$test_script" "$test_script" >> summary.log || ((number_of_failures++))
done
+echo '##############################################################################################################################'
+echo '## K 6 P E R F O R M A N C E T E S T R E S U L T S ##'
+echo '##############################################################################################################################'
+awk -F$'\t' '{printf "%-40s%-50s%-20s%-10s%-6s\n", $1, $2, $3, $4, $5}' summary.log
+
popd || exit 1
echo "NCMP TEST FAILURES: $number_of_failures"
diff --git a/k6-tests/ncmp/search-base.js b/k6-tests/ncmp/search-base.js
index 9c1b247..19a6a5a 100644
--- a/k6-tests/ncmp/search-base.js
+++ b/k6-tests/ncmp/search-base.js
@@ -24,7 +24,7 @@
export function searchRequest(searchType, searchFilter) {
const response = http.post(NCMP_BASE_URL + '/ncmp/v1/ch/' + searchType, searchFilter, {
- headers: { 'Content-Type': 'application/json' },
+ headers: {'Content-Type': 'application/json'},
});
check(response, {
'status equals 200': (r) => r.status === 200,
diff --git a/k6-tests/ncmp/utils.js b/k6-tests/ncmp/utils.js
index 3b61b21..18a8940 100644
--- a/k6-tests/ncmp/utils.js
+++ b/k6-tests/ncmp/utils.js
@@ -39,3 +39,23 @@
export function getRandomCmHandleId() {
return 'ch-' + (Math.floor(Math.random() * TOTAL_CM_HANDLES) + 1);
}
+
+function removeBracketsAndQuotes(str) {
+ return str.replace(/\[|\]|"/g, '');
+}
+
+export function makeCustomSummaryReport(data, options) {
+ const moduleName = `${__ENV.K6_MODULE_NAME}`;
+ let body = ``;
+ for (const condition in options.thresholds) {
+ let limit = JSON.stringify(options.thresholds[condition])
+ limit = removeBracketsAndQuotes(limit)
+ let limitKey = limit.split(' ')[0]
+ const actual = Math.ceil(data.metrics[condition].values[limitKey])
+ const result = data.metrics[condition].thresholds[limit].ok ? 'PASS' : 'FAIL'
+ const row = `${moduleName}\t${condition}\t${limit}\t${actual}\t${result}\n`;
+ body += row;
+ }
+ return body;
+}
+