k6-tests/ncmp/ncmp-kpi.js

/*
 *  ============LICENSE_START=======================================================
 *  Copyright (C) 2024 Nordix Foundation
 *  ================================================================================
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  SPDX-License-Identifier: Apache-2.0
 *  ============LICENSE_END=========================================================
 */

import { check } from 'k6';
import { Trend } from 'k6/metrics';
import { Reader } from 'k6/x/kafka';
import {
    TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS,
    makeCustomSummaryReport, makeBatchOfCmHandleIds, LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE,
    LEGACY_BATCH_TOPIC_NAME, KAFKA_BOOTSTRAP_SERVERS, REGISTRATION_BATCH_SIZE,
    LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS
} from './common/utils.js';
import { createCmHandles, deleteCmHandles, waitForAllCmHandlesToBeReady } from './common/cmhandle-crud.js';
import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
import { passthroughRead, passthroughWrite, legacyBatchRead } from './common/passthrough-crud.js';

let cmHandlesCreatedPerSecondTrend = new Trend('cmhandles_created_per_second', false);
let cmHandlesDeletedPerSecondTrend = new Trend('cmhandles_deleted_per_second', false);
let passthroughReadNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_read', true);
let passthroughReadNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_read_alt_id', true);
let passthroughWriteNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_write', true);
let passthroughWriteNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_write_alt_id', true);
let idSearchNoFilterDurationTrend = new Trend('id_search_nofilter_duration', true);
let idSearchModuleDurationTrend = new Trend('id_search_module_duration', true);
let idSearchPropertyDurationTrend = new Trend('id_search_property_duration', true);
let idSearchCpsPathDurationTrend = new Trend('id_search_cpspath_duration', true);
let idSearchTrustLevelDurationTrend = new Trend('id_search_trustlevel_duration', true);
let cmSearchNoFilterDurationTrend = new Trend('cm_search_nofilter_duration', true);
let cmSearchModuleDurationTrend = new Trend('cm_search_module_duration', true);
let cmSearchPropertyDurationTrend = new Trend('cm_search_property_duration', true);
let cmSearchCpsPathDurationTrend = new Trend('cm_search_cpspath_duration', true);
let cmSearchTrustLevelDurationTrend = new Trend('cm_search_trustlevel_duration', true);
let legacyBatchReadCmHandlesPerSecondTrend = new Trend('legacy_batch_read_cmhandles_per_second', false);

const legacyBatchEventReader = new Reader({
    brokers: KAFKA_BOOTSTRAP_SERVERS,
    topic: LEGACY_BATCH_TOPIC_NAME,
});

const DURATION = '15m';
const LEGACY_BATCH_THROUGHPUT_TEST_START_TIME = '15m30s';

export const options = {
    setupTimeout: '8m',
    teardownTimeout: '6m',
    scenarios: {
        passthrough_read_scenario: {
            executor: 'constant-vus',
            exec: 'passthroughReadScenario',
            vus: 2,
            duration: DURATION,
        },
        passthrough_read_alt_id_scenario: {
            executor: 'constant-vus',
            exec: 'passthroughReadAltIdScenario',
            vus: 2,
            duration: DURATION,
        },
        passthrough_write_scenario: {
            executor: 'constant-vus',
            exec: 'passthroughWriteScenario',
            vus: 2,
            duration: DURATION,
        },
        passthrough_write_alt_id_scenario: {
            executor: 'constant-vus',
            exec: 'passthroughWriteAltIdScenario',
            vus: 2,
            duration: DURATION,
        },
        cm_handle_id_search_nofilter_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleIdSearchNoFilterScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_search_nofilter_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleSearchNoFilterScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_id_search_module_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleIdSearchModuleScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_search_module_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleSearchModuleScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_id_search_property_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleIdSearchPropertyScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_search_property_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleSearchPropertyScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_id_search_cpspath_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleIdSearchCpsPathScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_search_cpspath_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleSearchCpsPathScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_id_search_trustlevel_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleIdSearchTrustLevelScenario',
            vus: 1,
            duration: DURATION,
        },
        cm_handle_search_trustlevel_scenario: {
            executor: 'constant-vus',
            exec: 'cmHandleSearchTrustLevelScenario',
            vus: 1,
            duration: DURATION,
        },
        legacy_batch_produce_scenario: {
            executor: 'shared-iterations',
            exec: 'legacyBatchProduceScenario',
            vus: 2,
            iterations: LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS,
            startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME,
        },
        legacy_batch_consume_scenario: {
            executor: 'per-vu-iterations',
            exec: 'legacyBatchConsumeScenario',
            vus: 1,
            iterations: 1,
            startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME,
        }
    },
    thresholds: {
        'http_req_failed': ['rate == 0'],
        'cmhandles_created_per_second': ['avg >= 22'],
        'cmhandles_deleted_per_second': ['avg >= 22'],
        'ncmp_overhead_passthrough_read': ['avg <= 40'],
        'ncmp_overhead_passthrough_write': ['avg <= 40'],
        'ncmp_overhead_passthrough_read_alt_id': ['avg <= 40'],
        'ncmp_overhead_passthrough_write_alt_id': ['avg <= 40'],
        'id_search_nofilter_duration': ['avg <= 2000'],
        'id_search_module_duration': ['avg <= 2000'],
        'id_search_property_duration': ['avg <= 2000'],
        'id_search_cpspath_duration': ['avg <= 2000'],
        'id_search_trustlevel_duration': ['avg <= 2000'],
        'cm_search_nofilter_duration': ['avg <= 15000'],
        'cm_search_module_duration': ['avg <= 15000'],
        'cm_search_property_duration': ['avg <= 15000'],
        'cm_search_cpspath_duration': ['avg <= 15000'],
        'cm_search_trustlevel_duration': ['avg <= 15000'],
        'legacy_batch_read_cmhandles_per_second': ['avg >= 150'],
    },
};

export function setup() {
    const startTimeInMillis = Date.now();

    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
        const response = createCmHandles(nextBatchOfCmHandleIds);
        check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 });
    }

    waitForAllCmHandlesToBeReady();

    const endTimeInMillis = Date.now();
    const totalRegistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;

    cmHandlesCreatedPerSecondTrend.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
}

export function teardown() {
    const startTimeInMillis = Date.now();

    let DEREGISTERED_CM_HANDLES = 0
    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
        const response = deleteCmHandles(nextBatchOfCmHandleIds);
        if (response.error_code === 0) {
              DEREGISTERED_CM_HANDLES += REGISTRATION_BATCH_SIZE
        }
        check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 });
    }

    const endTimeInMillis = Date.now();
    const totalDeregistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;

    cmHandlesDeletedPerSecondTrend.add(DEREGISTERED_CM_HANDLES / totalDeregistrationTimeInSeconds);
}

export function passthroughReadScenario() {
    const response = passthroughRead(false);
    if (check(response, { 'passthrough read status equals 200': (r) => r.status === 200 })) {
        const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS;
        passthroughReadNcmpOverheadTrend.add(overhead);
    }
}

export function passthroughReadAltIdScenario() {
    const response = passthroughRead(true);
    if (check(response, { 'passthrough read with alternate Id status equals 200': (r) => r.status === 200 })) {
        const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS;
        passthroughReadNcmpOverheadTrendWithAlternateId.add(overhead);
    }
}

export function passthroughWriteScenario() {
    const response = passthroughWrite(false);
    if (check(response, { 'passthrough write status equals 201': (r) => r.status === 201 })) {
        const overhead = response.timings.duration - WRITE_DATA_FOR_CM_HANDLE_DELAY_MS;
        passthroughWriteNcmpOverheadTrend.add(overhead);
    }
}

export function passthroughWriteAltIdScenario() {
    const response = passthroughWrite(true);
    if (check(response, { 'passthrough write with alternate Id status equals 201': (r) => r.status === 201 })) {
        const overhead = response.timings.duration - WRITE_DATA_FOR_CM_HANDLE_DELAY_MS;
        passthroughWriteNcmpOverheadTrendWithAlternateId.add(overhead);
    }
}

export function cmHandleIdSearchNoFilterScenario() {
    const response = executeCmHandleIdSearch('no-filter');
    if (check(response, { 'CM handle ID no-filter search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle ID no-filter search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        idSearchNoFilterDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleSearchNoFilterScenario() {
    const response = executeCmHandleSearch('no-filter');
    if (check(response, { 'CM handle no-filter search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle no-filter search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        cmSearchNoFilterDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleIdSearchModuleScenario() {
    const response = executeCmHandleIdSearch('module');
    if (check(response, { 'CM handle ID module search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle ID module search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        idSearchModuleDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleSearchModuleScenario() {
    const response = executeCmHandleSearch('module');
    if (check(response, { 'CM handle module search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle module search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        cmSearchModuleDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleIdSearchPropertyScenario() {
    const response = executeCmHandleIdSearch('property');
    if (check(response, { 'CM handle ID property search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle ID property search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        idSearchPropertyDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleSearchPropertyScenario() {
    const response = executeCmHandleSearch('property');
    if (check(response, { 'CM handle property search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle property search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        cmSearchPropertyDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleIdSearchCpsPathScenario() {
    const response = executeCmHandleIdSearch('cps-path-for-ready-cm-handles');
    if (check(response, { 'CM handle ID cps path search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle ID cps path search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        idSearchCpsPathDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleSearchCpsPathScenario() {
    const response = executeCmHandleSearch('cps-path-for-ready-cm-handles');
    if (check(response, { 'CM handle cps path search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle cps path search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        cmSearchCpsPathDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleIdSearchTrustLevelScenario() {
    const response = executeCmHandleIdSearch('trust-level');
    if (check(response, { 'CM handle ID trust level search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle ID trust level search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        idSearchTrustLevelDurationTrend.add(response.timings.duration);
    }
}

export function cmHandleSearchTrustLevelScenario() {
    const response = executeCmHandleSearch('trust-level');
    if (check(response, { 'CM handle trust level search status equals 200': (r) => r.status === 200 })
     && check(response, { 'CM handle trust level search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
        cmSearchTrustLevelDurationTrend.add(response.timings.duration);
    }
}

export function legacyBatchProduceScenario() {
    const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE, 0);
    const response = legacyBatchRead(nextBatchOfCmHandleIds);
    check(response, { 'data operation batch read status equals 200': (r) => r.status === 200 });
}

export function legacyBatchConsumeScenario() {
    const TOTAL_MESSAGES_TO_CONSUME = LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS * LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE;
    try {
        let messagesConsumed = 0;
        let startTime = Date.now();

        while (messagesConsumed < TOTAL_MESSAGES_TO_CONSUME) {
            let messages = legacyBatchEventReader.consume({ limit: 1000 });

            if (messages.length > 0) {
                messagesConsumed += messages.length;
            }
        }

        let endTime = Date.now();
        const timeToConsumeMessagesInSeconds = (endTime - startTime) / 1000.0;
        legacyBatchReadCmHandlesPerSecondTrend.add(TOTAL_MESSAGES_TO_CONSUME / timeToConsumeMessagesInSeconds);
    } catch (error) {
        legacyBatchReadCmHandlesPerSecondTrend.add(0);
        console.error(error);
    }
}

export function handleSummary(data) {
    return {
        stdout: makeCustomSummaryReport(data, options),
    };
}