Clean up the k6 test suite
- Remove the tests that are not in the Characteristics
- Set .625 seconds/operation for cm handle id search with module filter
- Set 13 seconds/operation for cm handle search with module filter
Issue-ID: CPS-2286
Change-Id: I88d5773abadb29ce46914029332384ae8e55e032
Signed-off-by: halil.cakal <halil.cakal@est.tech>
diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/10-mixed-load-test.js
index afa91af..a6b5b01 100644
--- a/k6-tests/ncmp/10-mixed-load-test.js
+++ b/k6-tests/ncmp/10-mixed-load-test.js
@@ -22,25 +22,27 @@
import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
import { passthroughRead } from './common/passthrough-read.js';
+const DURATION = '15m';
+
export const options = {
scenarios: {
passthrough_read: {
executor: 'constant-vus',
exec: 'passthrough_read',
vus: 10,
- duration: '1m',
+ duration: DURATION,
},
id_search_module: {
executor: 'constant-vus',
exec: 'id_search_module',
- vus: 5,
- duration: '1m',
+ vus: 3,
+ duration: DURATION,
},
cm_search_module: {
executor: 'constant-vus',
exec: 'cm_search_module',
- vus: 4,
- duration: '1m',
+ vus: 3,
+ duration: DURATION,
},
},
@@ -49,8 +51,8 @@
'http_req_failed{scenario:id_search_module}': ['rate == 0'],
'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
'http_req_duration{scenario:passthrough_read}': ['avg <= 2600'], // DMI delay + 100 ms
- 'http_req_duration{scenario:id_search_module}': ['avg <= 500'],
- 'http_req_duration{scenario:cm_search_module}': ['avg <= 30000'],
+ 'http_req_duration{scenario:id_search_module}': ['avg <= 625'],
+ 'http_req_duration{scenario:cm_search_module}': ['avg <= 13000'],
},
};
diff --git a/k6-tests/ncmp/3-passthrough-read.js b/k6-tests/ncmp/3-passthrough-read.js
deleted file mode 100644
index 39eb4ad..0000000
--- a/k6-tests/ncmp/3-passthrough-read.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { Trend } from 'k6/metrics';
-import { passthroughRead } from './common/passthrough-read.js'
-import { makeCustomSummaryReport } from './common/utils.js'
-
-let ncmpOverheadTrend = new Trend("ncmp_overhead");
-
-export const options = {
- vus: 12,
- duration: '30s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- ncmp_overhead: ['avg <= 50'],
- },
-};
-
-// The function that defines VU logic.
-export default function () {
- const response = passthroughRead();
- // Calculate overhead assuming DMI data delay is 2500ms.
- const dmiDelay = 2500; // This should be same as value DATA_FOR_CM_HANDLE_DELAY_MS in docker-compose.yml
- const overhead = response.timings.duration - dmiDelay;
- ncmpOverheadTrend.add(overhead);
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/4-id-search-no-filter.js b/k6-tests/ncmp/4-id-search-no-filter.js
deleted file mode 100644
index 3863b5d..0000000
--- a/k6-tests/ncmp/4-id-search-no-filter.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { executeCmHandleIdSearch } from './common/search-base.js';
-import { makeCustomSummaryReport } from './common/utils.js';
-
-export const options = {
- vus: 5,
- duration: '30s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 700'],
- },
-};
-
-export default function () {
- executeCmHandleIdSearch('no-filter');
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/5-search-no-filter.js b/k6-tests/ncmp/5-search-no-filter.js
deleted file mode 100644
index 67c9d59..0000000
--- a/k6-tests/ncmp/5-search-no-filter.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { executeCmHandleSearch } from './common/search-base.js';
-import { makeCustomSummaryReport } from './common/utils.js';
-
-export const options = {
- vus: 4,
- duration: '60s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 20000'],
- },
-};
-
-export default function () {
- executeCmHandleSearch('no-filter');
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/6-id-search-public-property.js b/k6-tests/ncmp/6-id-search-public-property.js
deleted file mode 100644
index 25bffaf..0000000
--- a/k6-tests/ncmp/6-id-search-public-property.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { executeCmHandleIdSearch } from './common/search-base.js';
-import { makeCustomSummaryReport } from './common/utils.js';
-
-export const options = {
- vus: 5,
- duration: '30s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 4500'],
- },
-};
-
-export default function () {
- executeCmHandleIdSearch('property');
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/7-search-public-property.js b/k6-tests/ncmp/7-search-public-property.js
deleted file mode 100644
index 53f069d..0000000
--- a/k6-tests/ncmp/7-search-public-property.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { executeCmHandleSearch } from './common/search-base.js';
-import { makeCustomSummaryReport } from './common/utils.js';
-
-export const options = {
- vus: 4,
- duration: '60s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 25000'],
- },
-};
-
-export default function () {
- executeCmHandleSearch('property');
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/8-id-search-module.js b/k6-tests/ncmp/8-id-search-module.js
deleted file mode 100644
index 8200ea2..0000000
--- a/k6-tests/ncmp/8-id-search-module.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { executeCmHandleIdSearch } from './common/search-base.js';
-import { makeCustomSummaryReport } from './common/utils.js';
-
-export const options = {
- vus: 5,
- duration: '30s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 200'],
- },
-};
-
-export default function () {
- executeCmHandleIdSearch('module');
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/9-search-module.js b/k6-tests/ncmp/9-search-module.js
deleted file mode 100644
index eafef99..0000000
--- a/k6-tests/ncmp/9-search-module.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2024 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-import { executeCmHandleSearch } from './common/search-base.js';
-import { makeCustomSummaryReport } from './common/utils.js';
-
-export const options = {
- vus: 4,
- duration: '60s',
- thresholds: {
- http_req_failed: ['rate == 0'],
- http_req_duration: ['avg <= 20000'],
- },
-};
-
-export default function () {
- executeCmHandleSearch('module');
-}
-
-export function handleSummary(data) {
- return {
- stdout: makeCustomSummaryReport(data, options),
- };
-}
diff --git a/k6-tests/ncmp/common/search-base.js b/k6-tests/ncmp/common/search-base.js
index f833a53..816baca 100644
--- a/k6-tests/ncmp/common/search-base.js
+++ b/k6-tests/ncmp/common/search-base.js
@@ -23,7 +23,6 @@
import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
const SEARCH_PARAMETERS_PER_SCENARIO = {
- 'no-filter': {},
'module': {
'cmHandleQueryParameters': [
{
@@ -31,14 +30,6 @@
'conditionParameters': [{'moduleName': 'ietf-yang-types-1'}]
}
]
- },
- 'property': {
- 'cmHandleQueryParameters': [
- {
- 'conditionName': 'hasAllProperties',
- 'conditionParameters': [{'Color': 'yellow'}]
- }
- ]
}
};
diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index f67b6ef..bf63709 100755
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -18,25 +18,12 @@
ALL_TEST_SCRIPTS=( \
1-create-cmhandles.js \
2-wait-for-cmhandles-to-be-ready.js \
-3-passthrough-read.js \
-4-id-search-no-filter.js \
-5-search-no-filter.js \
-6-id-search-public-property.js \
-7-search-public-property.js \
-8-id-search-module.js \
-9-search-module.js \
10-mixed-load-test.js \
11-delete-cmhandles.js \
)
pushd "$(dirname "$0")" || exit 1
-echo 'Warming up JVM (warmup results will not be recorded)'
-for test_script in "${ALL_TEST_SCRIPTS[@]}"; do
- echo "[warmup] k6 run $test_script"
- k6 --quiet -e TOTAL_CM_HANDLES=1000 -e REGISTRATION_BATCH_SIZE=10 -e K6_MODULE_NAME="$test_script" run "$test_script" > /dev/null
-done
-
printf "Test Case\tCondition\tLimit\tActual\tResult\n" > summary.log
number_of_failures=0