Add policy downloader
Add policy downloader to sync all policy types in the database
Issue-ID: CLAMP-518
Change-Id: I5ab82970cd3403e46fe7cc8447766977b11b68e7
Signed-off-by: sebdet <sebastien.determe@intl.att.com>
diff --git a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
index d99e9b5..9629420 100644
--- a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
+++ b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
@@ -30,6 +30,8 @@
import org.apache.camel.Exchange;
import org.apache.camel.builder.ExchangeBuilder;
import org.onap.clamp.clds.config.ClampProperties;
+import org.onap.clamp.clds.sdc.controller.installer.BlueprintMicroService;
+import org.onap.clamp.loop.template.PolicyModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -43,32 +45,52 @@
public class PolicyEngineServices {
private final CamelContext camelContext;
- private final ClampProperties refProp;
+ private static final EELFLogger logger = EELFManager.getInstance().getLogger(PolicyEngineServices.class);
+ private static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
+ private static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
+ private static int retryInterval = 0;
+ private static int retryLimit = 1;
- protected static final EELFLogger logger = EELFManager.getInstance().getLogger(PolicyEngineServices.class);
- protected static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
- protected static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
public static final String POLICY_RETRY_INTERVAL = "policy.retry.interval";
public static final String POLICY_RETRY_LIMIT = "policy.retry.limit";
@Autowired
public PolicyEngineServices(CamelContext camelContext, ClampProperties refProp) {
- this.refProp = refProp;
this.camelContext = camelContext;
+
+ if (refProp.getStringValue(POLICY_RETRY_LIMIT) != null) {
+ retryLimit = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_LIMIT));
+ }
+ if (refProp.getStringValue(POLICY_RETRY_INTERVAL) != null) {
+ retryInterval = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_INTERVAL));
+ }
}
- private void downloadAllPolicies() {
- /*
- * Exchange myCamelExchange = ExchangeBuilder.anExchange(camelContext)
- * .withProperty("blueprintResourceId",
- * resourceUuid).withProperty("blueprintServiceId", serviceUuid)
- * .withProperty("blueprintName", artifactName).build();
- * metricsLogger.info("Attempt n°" + i + " to contact DCAE inventory");
- *
- * Exchange exchangeResponse =
- * camelContext.createProducerTemplate().send("direct:get-all-policy-models",
- * myCamelExchange);
- */
+ public PolicyModel createPolicyModelFromPolicyEngine(String policyType, String policyVersion)
+ throws InterruptedException {
+ return new PolicyModel(policyType, this.downloadOnePolicy(policyType, policyVersion), policyVersion,
+ createPolicyAcronym(policyType));
+ }
+
+ public PolicyModel createPolicyModelFromPolicyEngine(BlueprintMicroService microService)
+ throws InterruptedException {
+ return createPolicyModelFromPolicyEngine(microService.getModelType(), microService.getModelVersion());
+ }
+
+ private static String createPolicyAcronym(String policyType) {
+ String[] policyNameArray = policyType.split("\\.");
+ return policyNameArray[policyNameArray.length - 1];
+ }
+
+ /**
+ * This method can be used to download all policy types + data types defined in
+ * policy engine.
+ *
+ * @return A yaml containing all policy Types and all data types
+ * @throws InterruptedException In case of issue when sleeping during the retry
+ */
+ public String downloadAllPolicies() throws InterruptedException {
+ return callCamelRoute(ExchangeBuilder.anExchange(camelContext).build(), "direct:get-all-policy-models");
}
/**
@@ -77,34 +99,24 @@
* @param policyType The policy type (id)
* @param policyVersion The policy version
* @return A string with the whole policy tosca model
- * @throws InterruptedException in case of issue when sleeping during the retry
+ * @throws InterruptedException In case of issue when sleeping during the retry
*/
public String downloadOnePolicy(String policyType, String policyVersion) throws InterruptedException {
- int retryInterval = 0;
- int retryLimit = 1;
- if (refProp.getStringValue(POLICY_RETRY_LIMIT) != null) {
- retryLimit = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_LIMIT));
- }
- if (refProp.getStringValue(POLICY_RETRY_INTERVAL) != null) {
- retryInterval = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_INTERVAL));
- }
+ return callCamelRoute(ExchangeBuilder.anExchange(camelContext).withProperty("policyModelName", policyType)
+ .withProperty("policyModelVersion", policyVersion).build(), "direct:get-policy-model");
+ }
+
+ private String callCamelRoute(Exchange exchange, String camelFlow) throws InterruptedException {
for (int i = 0; i < retryLimit; i++) {
- Exchange paramExchange = ExchangeBuilder.anExchange(camelContext)
- .withProperty("policyModelName", policyType).withProperty("policyModelVersion", policyVersion)
- .build();
-
- Exchange exchangeResponse = camelContext.createProducerTemplate().send("direct:get-policy-model",
- paramExchange);
-
+ Exchange exchangeResponse = camelContext.createProducerTemplate().send(camelFlow, exchange);
if (Integer.valueOf(200).equals(exchangeResponse.getIn().getHeader("CamelHttpResponseCode"))) {
return (String) exchangeResponse.getIn().getBody();
} else {
- logger.info("Policy " + retryInterval + "ms before retrying ...");
+ logger.info("Policy query " + retryInterval + "ms before retrying ...");
// wait for a while and try to connect to DCAE again
Thread.sleep(retryInterval);
}
}
return "";
}
-
}
diff --git a/src/main/java/org/onap/clamp/loop/CsarInstaller.java b/src/main/java/org/onap/clamp/loop/CsarInstaller.java
index 022b0e2..c0cfac9 100644
--- a/src/main/java/org/onap/clamp/loop/CsarInstaller.java
+++ b/src/main/java/org/onap/clamp/loop/CsarInstaller.java
@@ -183,21 +183,10 @@
return newSet;
}
- private static String createPolicyAcronym(String policyType) {
- String[] policyNameArray = policyType.split("\\.");
- return policyNameArray[policyNameArray.length - 1];
- }
-
- private PolicyModel createPolicyModel(BlueprintMicroService microService) throws InterruptedException {
- return new PolicyModel(microService.getModelType(),
- policyEngineServices.downloadOnePolicy(microService.getModelType(), microService.getModelVersion()),
- microService.getModelVersion(), createPolicyAcronym(microService.getModelType()));
- }
-
private PolicyModel getPolicyModel(BlueprintMicroService microService) throws InterruptedException {
return policyModelsRepository
.findById(new PolicyModelId(microService.getModelType(), microService.getModelVersion()))
- .orElse(createPolicyModel(microService));
+ .orElse(policyEngineServices.createPolicyModelFromPolicyEngine(microService));
}
/**
diff --git a/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java b/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java
index b712dc3..8795a12 100644
--- a/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java
+++ b/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java
@@ -26,12 +26,18 @@
import com.att.eelf.configuration.EELFLogger;
import com.att.eelf.configuration.EELFManager;
-import org.apache.camel.CamelContext;
-import org.onap.clamp.clds.client.DcaeInventoryServices;
-import org.onap.clamp.clds.config.ClampProperties;
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+
+import org.onap.clamp.clds.client.PolicyEngineServices;
+import org.onap.clamp.loop.template.PolicyModel;
+import org.onap.clamp.loop.template.PolicyModelId;
+import org.onap.clamp.loop.template.PolicyModelsRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.yaml.snakeyaml.Yaml;
/**
* This class implements a periodic job that is done in the background to
@@ -42,20 +48,49 @@
@Profile("clamp-policy-controller")
public class PolicyDownloader {
- protected static final EELFLogger logger = EELFManager.getInstance().getLogger(DcaeInventoryServices.class);
+ protected static final EELFLogger logger = EELFManager.getInstance().getLogger(PolicyDownloader.class);
protected static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
protected static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
public static final String POLICY_RETRY_INTERVAL = "policy.retry.interval";
public static final String POLICY_RETRY_LIMIT = "policy.retry.limit";
- private final CamelContext camelContext;
-
- private final ClampProperties refProp;
+ private final PolicyEngineServices policyEngineServices;
+ private final PolicyModelsRepository policyModelsRepository;
@Autowired
- public PolicyDownloader(CamelContext camelContext, ClampProperties refProp) {
- this.refProp = refProp;
- this.camelContext = camelContext;
+ public PolicyDownloader(PolicyEngineServices policyEngineService, PolicyModelsRepository policyModelsRepository) {
+ this.policyEngineServices = policyEngineService;
+ this.policyModelsRepository = policyModelsRepository;
+ }
+
+ private void createPolicyInDbIfNeeded(PolicyModel policyModel) {
+ if (!policyModelsRepository
+ .existsById(new PolicyModelId(policyModel.getPolicyModelType(), policyModel.getVersion()))) {
+ policyModelsRepository.save(policyModel);
+ }
+ }
+
+ @Scheduled(fixedRate = 120000)
+ public void synchronizeAllPolicies() throws InterruptedException {
+ try {
+ LinkedHashMap<String, Object> loadedYaml = new Yaml().load(policyEngineServices.downloadAllPolicies());
+ if (loadedYaml == null || loadedYaml.isEmpty()) {
+ logger.warn(
+ "getAllPolicyType yaml returned by policy engine could not be decoded, as it's null or empty");
+ return;
+ }
+
+ LinkedHashMap<String, Object> policyTypesList = (LinkedHashMap<String, Object>) loadedYaml
+ .get("policy_types");
+ for (Entry<String, Object> policyType : policyTypesList.entrySet()) {
+ createPolicyInDbIfNeeded(policyEngineServices.createPolicyModelFromPolicyEngine(policyType.getKey(),
+ ((String) ((LinkedHashMap<String, Object>) policyType.getValue()).get("version"))));
+ }
+ } catch (InterruptedException e) {
+ logger.warn("query to policy engine has been interrupted", e);
+ throw e;
+ }
+
}
}
diff --git a/src/main/resources/application-noaaf.properties b/src/main/resources/application-noaaf.properties
index b9af1b4..69d1687 100644
--- a/src/main/resources/application-noaaf.properties
+++ b/src/main/resources/application-noaaf.properties
@@ -73,7 +73,7 @@
server.servlet.context-path=/
#Modified engine-rest applicationpath
-spring.profiles.active=clamp-default,clamp-default-user,clamp-sdc-controller,clamp-ssl-config
+spring.profiles.active=clamp-default,clamp-default-user,clamp-sdc-controller,clamp-ssl-config,clamp-policy-controller
spring.http.converters.preferred-json-mapper=gson
#The max number of active threads in this pool
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index e456899..bb25abf 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -78,7 +78,7 @@
server.servlet.context-path=/
#Modified engine-rest applicationpath
-spring.profiles.active=clamp-default,clamp-aaf-authentication,clamp-sdc-controller,clamp-ssl-config
+spring.profiles.active=clamp-default,clamp-aaf-authentication,clamp-sdc-controller,clamp-ssl-config,clamp-policy-controller
spring.http.converters.preferred-json-mapper=gson
#The max number of active threads in this pool
diff --git a/src/main/resources/clds/camel/routes/policy-flows.xml b/src/main/resources/clds/camel/routes/policy-flows.xml
index ce24b27..c28e454 100644
--- a/src/main/resources/clds/camel/routes/policy-flows.xml
+++ b/src/main/resources/clds/camel/routes/policy-flows.xml
@@ -140,7 +140,7 @@
<log loggingLevel="INFO"
message="Endpoint to get all policy models: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes"></log>
<toD
- uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
+ uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
<convertBodyTo type="java.lang.String"/>
<doFinally>
<to uri="direct:reset-raise-http-exception-flag"/>
@@ -174,7 +174,7 @@
<log loggingLevel="INFO"
message="Endpoint to get policy model: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}"></log>
<toD
- uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
+ uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
<convertBodyTo type="java.lang.String"/>
<doFinally>
<to uri="direct:reset-raise-http-exception-flag"/>
diff --git a/src/test/resources/clds/camel/routes/policy-flows.xml b/src/test/resources/clds/camel/routes/policy-flows.xml
index ce24b27..c28e454 100644
--- a/src/test/resources/clds/camel/routes/policy-flows.xml
+++ b/src/test/resources/clds/camel/routes/policy-flows.xml
@@ -140,7 +140,7 @@
<log loggingLevel="INFO"
message="Endpoint to get all policy models: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes"></log>
<toD
- uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
+ uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
<convertBodyTo type="java.lang.String"/>
<doFinally>
<to uri="direct:reset-raise-http-exception-flag"/>
@@ -174,7 +174,7 @@
<log loggingLevel="INFO"
message="Endpoint to get policy model: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}"></log>
<toD
- uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
+ uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&useSystemProperties=true&throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&authMethod=Basic&authUsername={{clamp.config.policy.api.userName}}&authPassword={{clamp.config.policy.api.password}}&connectionTimeToLive=5000&httpClient.connectTimeout=10000&httpClient.socketTimeout=20000&authenticationPreemptive=true&connectionClose=true"/>
<convertBodyTo type="java.lang.String"/>
<doFinally>
<to uri="direct:reset-raise-http-exception-flag"/>
diff --git a/src/test/resources/http-cache/example/policy/api/v1/policytypes/.file b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.file
new file mode 100644
index 0000000..7394d3f
--- /dev/null
+++ b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.file
@@ -0,0 +1,60 @@
+tosca_definitions_version: tosca_simple_yaml_1_0_0
+policy_types:
+ - onap.policies.Monitoring:
+ version: 1.0.0
+ description: A base policy type for all policies that govern monitoring provision
+ derived_from: tosca.policies.Root
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controlloop.Operational:
+ version: 1.0.0
+ description: Operational Policy for Control Loops
+ derived_from: tosca.policies.Root
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controloop.operational.Drools:
+ version: 1.0.0
+ description: Operational Policy for Control Loops using the Drools PDP
+ derived_from: onap.policies.controlloop.Operational
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controloop.operational.Apex:
+ version: 1.0.0
+ description: Operational Policy for Control Loops using the APEX PDP
+ derived_from: onap.policies.controlloop.Operational
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controlloop.Guard:
+ version: 1.0.0
+ description: Operational Policy for Control Loops
+ derived_from: tosca.policies.Root
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controlloop.guard.FrequencyLimiter:
+ version: 1.0.0
+ description: Supports limiting the frequency of actions being taken by a Actor.
+ derived_from: onap.policies.controlloop.Guard
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controlloop.guard.Blacklist:
+ version: 1.0.0
+ description: Supports blacklist of VNF's from performing control loop actions on.
+ derived_from: onap.policies.controlloop.Guard
+ properties:
+ # Omitted for brevity, see Section 1
+
+ - onap.policies.controlloop.guard.MinMax:
+ version: 1.0.0
+ description: Supports Min/Max number of VF Modules
+ derived_from: onap.policies.controlloop.Guard
+ properties:
+ # Omitted for brevity, see Section 1
+
+data_types:
+ # Any bespoke data types referenced by policy type definitions[]
diff --git a/src/test/resources/http-cache/example/policy/api/v1/policytypes/.header b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.header
new file mode 100644
index 0000000..6a280d9
--- /dev/null
+++ b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.header
@@ -0,0 +1 @@
+{"Transfer-Encoding": "chunked", "Set-Cookie": "JSESSIONID=158qxkdtdobkd1umr3ikkgrmlx;Path=/", "Expires": "Thu, 01 Jan 1970 00:00:00 GMT", "Server": "Jetty(9.3.21.v20170918)", "Content-Type": "application/json", "X-ECOMP-RequestID": "e2ddb3c8-994f-47df-b4dc-097d4fb55c08"}
\ No newline at end of file