Add SLAs validations to docker csit robot tests
Issue-ID: POLICY-4529
Change-Id: I197a51892e9135d20c2e8a49ce7787df3bfbd4fa
Signed-off-by: adheli.tavares <adheli.tavares@est.tech>
diff --git a/.gitignore b/.gitignore
index d4aa5c2..fee07ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@
.pydevproject
models
.vscode/
+**/*.log
diff --git a/csit/common-library.robot b/csit/common-library.robot
index e8a9640..871e7ef 100644
--- a/csit/common-library.robot
+++ b/csit/common-library.robot
@@ -120,7 +120,18 @@
Log Received response from policy ${resp.text}
[return] ${resp}
-QueryPrometheus ${query}
- ${resp}= GET http://localhost:30259/api/v1/query?query=${query} expected_status=200
- Log Received response from policy ${resp.text}
- [return] ${resp}
+QueryPrometheus
+ [Arguments] ${query}
+ ${params}= Create Dictionary query=${query}
+ ${resp}= GET http://localhost:30259/api/v1/query ${params}
+ Status Should Be OK
+ Log Received response from Prometheus ${resp.text}
+ [return] ${resp.json()}
+
+ValidateResponseTime
+ [Arguments] ${job} ${uri} ${method} ${timeLimit}
+ [Documentation] Check if uri response is under the required time
+ ${resp}= QueryPrometheus http_server_requests_seconds_sum{uri="${uri}",method="${method}",job="${job}"}/http_server_requests_seconds_count{uri="${uri}",method="${method}",job="${job}"}
+ ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]}
+ ${actualTime}= Set Variable ${rawNumber * ${1000}}
+ Should Be True ${actualTime} <= ${timeLimit}
diff --git a/csit/pap/plans/setup.sh b/csit/pap/plans/setup.sh
index b656ef5..4c77acc 100755
--- a/csit/pap/plans/setup.sh
+++ b/csit/pap/plans/setup.sh
@@ -37,15 +37,8 @@
sleep 10
unset http_proxy https_proxy
-POLICY_PAP_IP=$(get-instance-ip.sh policy-pap)
POLICY_PAP_PORT=30442
-POLICY_API_IP=$(get-instance-ip.sh policy-api)
POLICY_API_PORT=30440
-MARIADB_IP=$(get-instance-ip.sh mariadb)
-
-echo PAP IP IS "${POLICY_PAP_IP}"
-echo API IP IS "${POLICY_API_IP}"
-echo MARIADB IP IS "${MARIADB_IP}"
# wait for the app to start up
"${SCRIPTS}"/wait_for_rest.sh localhost "${POLICY_PAP_PORT}"
@@ -55,8 +48,6 @@
NODETEMPLATES=${WORKSPACE}/models/models-examples/src/main/resources/nodetemplates
ROBOT_VARIABLES=""
-ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_PAP_IP:${POLICY_PAP_IP}"
-ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_API_IP:${POLICY_API_IP}"
ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_PAP_PORT:${POLICY_PAP_PORT}"
ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_API_PORT:${POLICY_API_PORT}"
ROBOT_VARIABLES="${ROBOT_VARIABLES} -v DATA:${DATA}"
diff --git a/csit/pap/plans/testplan.txt b/csit/pap/plans/testplan.txt
index 1e5b1f9..4fac102 100644
--- a/csit/pap/plans/testplan.txt
+++ b/csit/pap/plans/testplan.txt
@@ -1,3 +1,4 @@
# Test suites are relative paths under [policy/docker.git]/csit/[project]/tests.
# Place the suites in run order.
pap-test.robot
+pap-slas.robot
diff --git a/csit/pap/tests/pap-slas.robot b/csit/pap/tests/pap-slas.robot
new file mode 100644
index 0000000..045530a
--- /dev/null
+++ b/csit/pap/tests/pap-slas.robot
@@ -0,0 +1,60 @@
+*** Settings ***
+Library Collections
+Library RequestsLibrary
+Library OperatingSystem
+Library json
+Resource ${CURDIR}/../../common-library.robot
+
+*** Keywords ***
+ValidateResponseTimeForPap
+ [Arguments] ${uri} ${method}
+ [Documentation] Check if uri response is under the 500ms required time for pap metrics
+ ValidateResponseTime pap-metrics ${uri} ${method} 500
+
+*** Test Cases ***
+WaitForPrometheusServer
+ [Documentation] Sleep time to wait for Prometheus server to gather all metrics
+ Sleep 1 minute
+
+ValidateResponseTimeForHealthcheck
+ [Documentation] Validate component healthcheck response time
+ ValidateResponseTimeForPap /healthcheck GET
+
+ValidateResponseTimeForSystemHealthcheck
+ [Documentation] Validate if system healthcheck response time is under 1000ms
+ ValidateResponseTime pap-metrics /components/healthcheck GET 10000
+
+ValidateResponseTimeForStatistics
+ [Documentation] Validate statistics response time
+ ValidateResponseTimeForPap /statistics GET
+
+# TODO: includes notification, so always over 500ms
+# ValidateResponseTimeCreateGroup
+# [Documentation] Validate create group response time
+# ValidateResponseTimeForPap /pdps/groups/batch POST
+
+ValidateResponseTimeQueryPolicyAudit
+ [Documentation] Validate query audits response time
+ ValidateResponseTimeForPap /policies/audit GET
+
+ValidateResponseTimeUpdateGroup
+ [Documentation] Validate pdps/group response time
+ ValidateResponseTimeForPap /pdps/groups/{name} PUT
+
+ValidatePolicyDeploymentTime
+ [Documentation] Check if deployment of policy is under 2000ms
+ ${resp}= QueryPrometheus pap_policy_deployments_seconds_sum{operation="deploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="deploy",status="SUCCESS"}
+ ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]}
+ ${actualTime}= Set Variable ${rawNumber * ${1000}}
+ Should Be True ${actualTime} <= ${2000}
+
+ValidateResponseTimeDeletePolicy
+ [Documentation] Check if undeployment of policy is under 2000ms
+ ${resp}= QueryPrometheus pap_policy_deployments_seconds_sum{operation="undeploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="undeploy",status="SUCCESS"}
+ ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]}
+ ${actualTime}= Set Variable ${rawNumber * ${1000}}
+ Should Be True ${actualTime} <= ${2000}
+
+ValidateResponseTimeDeleteGroup
+ [Documentation] Validate delete group response time
+ ValidateResponseTimeForPap /pdps/groups/{name} DELETE
diff --git a/csit/pap/tests/pap-test.robot b/csit/pap/tests/pap-test.robot
index c417a83..a4f0853 100644
--- a/csit/pap/tests/pap-test.robot
+++ b/csit/pap/tests/pap-test.robot
@@ -12,6 +12,18 @@
${resp}= PerformGetRequest ${POLICY_PAP_PORT} ${url} 200 null ${auth}
[return] ${resp}
+ValidateResponseTimeForPap
+ [Arguments] ${uri} ${method}
+ [Documentation] Check if uri response is under the required time for pap metrics
+ ValidateResponseTime pap-metrics ${uri} ${method} 500
+
+ValidateDeploymentTime
+ [Documentation] Check if deployment of policy is under 2000ms
+ ${resp}= QueryPrometheus pap_policy_deployments_seconds_sum{operation="deploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="deploy",status="SUCCESS"}
+ ${rawNumber}= Evaluate ${resp['data']['result'][0]['value'][1]}
+ ${actualTime}= Set Variable ${rawNumber * ${1000}}
+ Should Be True ${actualTime} <= ${2000}
+
*** Test Cases ***
LoadPolicy
[Documentation] Create a policy named 'onap.restart.tca' and version '1.0.0' using specific api
@@ -126,3 +138,16 @@
QueryPdpGroupsAfterDelete
[Documentation] Verify PdpGroups after delete
QueryPdpGroups 1 defaultGroup ACTIVE 0 null null null
+
+# ValidateSlaForPap
+# [Documentation] Run checks against Prometheus server to check response time
+# Sleep 30s
+# ValidateDeploymentTime
+# ValidateResponseTime pap-metrics /components/healthcheck GET 10000
+# ValidateResponseTimeForPap /healthcheck GET
+# ValidateResponseTimeForPap /statistics GET
+# ValidateResponseTimeForPap /policies/audit GET
+# ValidateResponseTimeForPap /pdps/groups/{name} PUT
+# ValidateResponseTimeForPap /pdps/policies/{name} DELETE
+# ValidateResponseTimeForPap /pdps/groups/{name} DELETE
+# ValidateResponseTimeForPap /pdps/groups/batch POST
diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh
index a3837b0..0f15c75 100755
--- a/csit/run-project-csit.sh
+++ b/csit/run-project-csit.sh
@@ -27,17 +27,18 @@
function on_exit(){
rc=$?
if [[ ${WORKSPACE} ]]; then
- if [[ ${WORKDIR} ]]; then
- rsync -av "${WORKDIR}/" "${WORKSPACE}/csit/archives/${PROJECT}"
- fi
# Record list of active docker containers
- docker ps
+ docker ps --format "table {{ .Names }}\t{{ .Status }}"
# Show the logs from all containers
- docker-compose -f "${WORKSPACE}/csit/docker-compose-all.yml" logs
+ docker-compose -f "${WORKSPACE}/csit/docker-compose-all.yml" logs > docker_compose.log
# show memory consumption after all docker instances initialized
docker_stats
+
+ if [[ ${WORKDIR} ]]; then
+ rsync -av "${WORKDIR}/" "${WORKSPACE}/csit/archives/${PROJECT}"
+ fi
fi
# Run teardown script plan if it exists
cd "${TESTPLANDIR}/plans/"
@@ -57,25 +58,20 @@
# General memory details
if [ "$(uname -s)" == "Darwin" ]
then
- echo "> top -l1 | head -10"
sh -c "top -l1 | head -10"
echo
else
- echo "> top -bn1 | head -3"
sh -c "top -bn1 | head -3"
echo
- echo "> free -h"
sh -c "free -h"
echo
fi
# Memory details per Docker
- echo "> docker ps"
- docker ps
+ docker ps --format "table {{ .Names }}\t{{ .Status }}"
echo
- echo "> docker stats --no-stream"
docker stats --no-stream
echo
}
diff --git a/csit/stop-all.sh b/csit/stop-all.sh
index 4e6b0aa..9adfcf3 100755
--- a/csit/stop-all.sh
+++ b/csit/stop-all.sh
@@ -23,4 +23,4 @@
source "${SCRIPTS}"/get-versions.sh
-docker-compose -f "${SCRIPTS}"/docker-compose-all.yml down
+docker-compose -f "${SCRIPTS}"/docker-compose-all.yml down -v
diff --git a/csit/wait_for_port.sh b/csit/wait_for_port.sh
index 5a8be25..6dcb3ca 100755
--- a/csit/wait_for_port.sh
+++ b/csit/wait_for_port.sh
@@ -1,7 +1,7 @@
#!/bin/sh
# ============LICENSE_START====================================================
# Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
-# Modifications Copyright (C) 2022 Nordix Foundation.
+# Modifications Copyright (C) 2022-2023 Nordix Foundation.
# =============================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -64,7 +64,7 @@
do
if command -v docker > /dev/null 2>&1
then
- docker ps
+ docker ps --format "table {{ .Names }}\t{{ .Status }}"
fi
nc -vz "$host" "$port"
diff --git a/csit/wait_for_rest.sh b/csit/wait_for_rest.sh
index bf47ccf..a64c306 100755
--- a/csit/wait_for_rest.sh
+++ b/csit/wait_for_rest.sh
@@ -54,7 +54,7 @@
do
if command -v docker > /dev/null 2>&1
then
- docker ps
+ docker ps --format "table {{ .Names }}\t{{ .Status }}"
fi
curl "http://$host:$port" > /dev/null 2>&1
rc=$?