Implement Delete timeouts

* Implements new logic around when instances are deleted. See flowcharts in docs/. Basically timeouts now trigger to actually delete instances from a1s database, and these timeouts are configurable.
* Eliminates the barrier to deleting an instance when no xapp evdr replied (via timeouts)
* Add two new ENV variables that control timeouts
* Make unit tests more modular so new workflows can be tested easily
* Changes the API for ../status to return a richer structure
* Clean up unused items in the integration tests helm chart
* Removed "RMR_RCV_RETRY_INTERVAL" leftovers since this isn't used
anymore

Issue-ID: RICPLT-2619
Signed-off-by: Tommy Carpenter <tc677g@att.com>
Change-Id: I5b2f9cc3a6e8da7fe636a4cde085a5e1a3770f62
diff --git a/integration_tests/test_a1.tavern.yaml b/integration_tests/test_a1.tavern.yaml
index b4ccd62..94fe0de 100644
--- a/integration_tests/test_a1.tavern.yaml
+++ b/integration_tests/test_a1.tavern.yaml
@@ -176,7 +176,9 @@
       method: GET
     response:
       status_code: 200
-      # tavern doesn't yet let you check string statuses!!!
+      body:
+        instance_status: "IN EFFECT"
+        has_been_deleted: False
 
   - name: instance list 200 and contains the instance
     request:
@@ -189,13 +191,25 @@
 
   # DELETE the instance and make sure subsequent GETs return properly
   - name: delete the instance
-    delay_after: 3 # give it a few seconds for rmr
+    delay_after: 4
     request:
       url: http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy
       method: DELETE
     response:
       status_code: 202
 
+  - name: status should now be not in effect but still there
+    delay_before: 3 # give it a few seconds for rmr
+    delay_after: 8 # 3 + 11 > 10; that is, wait until t2 expires
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20000/policies/admission_control_policy/status
+      method: GET
+    response:
+      status_code: 200
+      body:
+        instance_status: "NOT IN EFFECT"
+        has_been_deleted: True
+
   - name: instance list 200 but no instance
     request:
       url: http://localhost:10000/a1-p/policytypes/20000/policies
@@ -367,6 +381,16 @@
     response:
       status_code: 202
 
+  - name: test the delay status get, not in effect yet
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+      method: GET
+    response:
+      status_code: 200
+      body:
+        instance_status: "NOT IN EFFECT"
+        has_been_deleted: False
+
   - name: test the delay policy get
     request:
       url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest
@@ -376,16 +400,6 @@
       body:
         test: foo
 
-  - name: test the delay status get
-    max_retries: 3
-    delay_before: 6  # give it a few seconds for rmr ; delay reciever sleeps for 5 seconds by default
-    request:
-      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
-      method: GET
-    response:
-      status_code: 200
-      # tavern doesn't let you check non json yet!
-
   - name: instance list 200 and there
     request:
       url: http://localhost:10000/a1-p/policytypes/20001/policies
@@ -395,6 +409,55 @@
       body:
        - delaytest
 
+  - name: test the delay status get
+    max_retries: 3
+    delay_before: 6  # give it a few seconds for rmr ; delay reciever sleeps for 5 seconds by default
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+      method: GET
+    response:
+      status_code: 200
+      body:
+        instance_status: "IN EFFECT"
+        has_been_deleted: False
+
+  # DELETE the instance and make sure subsequent GETs return properly
+  - name: delete the instance
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest
+      method: DELETE
+    response:
+      status_code: 202
+
+  - name: test the delay status get immediately
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+      method: GET
+    response:
+      status_code: 200
+      body:
+        instance_status: "IN EFFECT"
+        has_been_deleted: True
+
+  - name: test the delay status get after delay but before timers
+    delay_before: 7
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+      method: GET
+    response:
+      status_code: 200
+      body:
+        instance_status: "NOT IN EFFECT"
+        has_been_deleted: True
+
+  - name: test the delay status get after delay and after the timers
+    delay_before: 7
+    request:
+      url: http://localhost:10000/a1-p/policytypes/20001/policies/delaytest/status
+      method: GET
+    response:
+      status_code: 404
+
 ---
 
 test_name: test bad routing file endpoint