Storing EiJobs persistently
Created Ei jobs can then survive a restart of the container/POD
Change-Id: Ib3fede385b58f394f55cde068ba2ec5e1f4b0ebc
Signed-off-by: PatrikBuhr <patrik.buhr@est.tech>
Issue-ID: NONRTRIC-173
diff --git a/enrichment-coordinator-service/Dockerfile b/enrichment-coordinator-service/Dockerfile
index 51d45b5..744a237 100644
--- a/enrichment-coordinator-service/Dockerfile
+++ b/enrichment-coordinator-service/Dockerfile
@@ -24,6 +24,8 @@
WORKDIR /opt/app/enrichment-coordinator-service
RUN mkdir -p /var/log/enrichment-coordinator-service
RUN mkdir -p /opt/app/enrichment-coordinator-service/etc/cert/
+RUN mkdir -p /var/enrichment-coordinator-service
+RUN chmod -R 777 /var/enrichment-coordinator-service
EXPOSE 8083 8434
diff --git a/enrichment-coordinator-service/config/application.yaml b/enrichment-coordinator-service/config/application.yaml
index e64db0c..850dc67 100644
--- a/enrichment-coordinator-service/config/application.yaml
+++ b/enrichment-coordinator-service/config/application.yaml
@@ -35,4 +35,5 @@
trust-store-used: false
trust-store-password: policy_agent
trust-store: /opt/app/enrichment-coordinator-service/etc/cert/truststore.jks
+ vardata-directory: /var/enrichment-coordinator-service
diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java
index f4cf9dc..c5d2bec 100644
--- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java
+++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java
@@ -22,11 +22,15 @@
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.lang.invoke.MethodHandles;
+
import org.apache.catalina.connector.Connector;
import org.oransc.enrichment.configuration.ApplicationConfig;
import org.oransc.enrichment.repository.EiJobs;
import org.oransc.enrichment.repository.EiProducers;
import org.oransc.enrichment.repository.EiTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
@@ -40,6 +44,7 @@
private int httpPort = 0;
private final ApplicationConfig applicationConfig = new ApplicationConfig();
+ private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Bean
public ObjectMapper mapper() {
@@ -57,7 +62,13 @@
@Bean
public EiJobs eiJobs() {
- return new EiJobs();
+ EiJobs jobs = new EiJobs(getApplicationConfig());
+ try {
+ jobs.restoreJobsFromDatabase();
+ } catch (Exception e) {
+ logger.error("Could not restore jobs from database: {}", e.getMessage());
+ }
+ return jobs;
}
@Bean
diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java
index 2d4087f..8937464 100644
--- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java
+++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java
@@ -36,6 +36,10 @@
@Value("${app.filepath}")
private String localConfigurationFilePath;
+ @Getter
+ @Value("${app.vardata-directory}")
+ private String vardataDirectory;
+
@Value("${server.ssl.key-store-type}")
private String sslKeyStoreType = "";
diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java
index 706c8dd..1532c53 100644
--- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java
+++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java
@@ -20,12 +20,29 @@
package org.oransc.enrichment.repository;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapterFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.ServiceLoader;
import java.util.Vector;
+import org.oransc.enrichment.configuration.ApplicationConfig;
import org.oransc.enrichment.exceptions.ServiceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.FileSystemUtils;
/**
* Dynamic representation of all existing EI jobs.
@@ -35,11 +52,29 @@
private MultiMap<EiJob> jobsByType = new MultiMap<>();
private MultiMap<EiJob> jobsByOwner = new MultiMap<>();
+ private final Gson gson;
+
+ private final ApplicationConfig config;
+ private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public EiJobs(ApplicationConfig config) {
+ this.config = config;
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ ServiceLoader.load(TypeAdapterFactory.class).forEach(gsonBuilder::registerTypeAdapterFactory);
+ this.gson = gsonBuilder.create();
+ }
+
+ public synchronized void restoreJobsFromDatabase() throws IOException {
+ File dbDir = new File(getDatabaseDirectory());
+ for (File file : dbDir.listFiles()) {
+ String json = Files.readString(file.toPath());
+ EiJob job = gson.fromJson(json, EiJob.class);
+ this.put(job, false);
+ }
+ }
public synchronized void put(EiJob job) {
- allEiJobs.put(job.id(), job);
- jobsByType.put(job.typeId(), job.id(), job);
- jobsByOwner.put(job.owner(), job.id(), job);
+ this.put(job, true);
}
public synchronized Collection<EiJob> getJobs() {
@@ -82,6 +117,13 @@
this.allEiJobs.remove(job.id());
jobsByType.remove(job.typeId(), job.id());
jobsByOwner.remove(job.owner(), job.id());
+
+ try {
+ Files.delete(getPath(job));
+ } catch (IOException e) {
+ logger.warn("Could not remove file: {}", e.getMessage());
+ }
+
}
public synchronized int size() {
@@ -92,6 +134,43 @@
this.allEiJobs.clear();
this.jobsByType.clear();
jobsByOwner.clear();
+ try {
+ FileSystemUtils.deleteRecursively(Path.of(getDatabaseDirectory()));
+ } catch (IOException e) {
+ logger.warn("Could not delete database : {}", e.getMessage());
+ }
+ }
+
+ private void put(EiJob job, boolean storePersistently) {
+ allEiJobs.put(job.id(), job);
+ jobsByType.put(job.typeId(), job.id(), job);
+ jobsByOwner.put(job.owner(), job.id(), job);
+ if (storePersistently) {
+ storeJobInFile(job);
+ }
+ }
+
+ private void storeJobInFile(EiJob job) {
+ try {
+ Files.createDirectories(Paths.get(getDatabaseDirectory()));
+ try (PrintStream out = new PrintStream(new FileOutputStream(getFile(job)))) {
+ out.print(gson.toJson(job));
+ }
+ } catch (Exception e) {
+ logger.warn("Could not save job: {} {}", job.id(), e.getMessage());
+ }
+ }
+
+ private File getFile(EiJob job) {
+ return getPath(job).toFile();
+ }
+
+ private Path getPath(EiJob job) {
+ return Path.of(getDatabaseDirectory(), job.id());
+ }
+
+ private String getDatabaseDirectory() {
+ return config.getVardataDirectory() + "/database";
}
}
diff --git a/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java b/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java
index e707fb7..30eaf68 100644
--- a/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java
+++ b/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java
@@ -88,7 +88,8 @@
@TestPropertySource(
properties = { //
"server.ssl.key-store=./config/keystore.jks", //
- "app.webclient.trust-store=./config/truststore.jks"})
+ "app.webclient.trust-store=./config/truststore.jks", //
+ "app.vardata-directory=./target"})
class ApplicationTest {
private final String EI_TYPE_ID = "typeId";
private final String EI_PRODUCER_ID = "producerId";
@@ -529,6 +530,33 @@
assertThat(resp.getBody()).contains("hunky dory");
}
+ @Test
+ void testEiJobDatabase() throws Exception {
+ putEiProducerWithOneType(EI_PRODUCER_ID, EI_TYPE_ID);
+ putEiJob(EI_TYPE_ID, "jobId1");
+ putEiJob(EI_TYPE_ID, "jobId2");
+
+ assertThat(this.eiJobs.size()).isEqualTo(2);
+
+ {
+ // Restore the jobs
+ EiJobs jobs = new EiJobs(this.applicationConfig);
+ jobs.restoreJobsFromDatabase();
+ assertThat(jobs.size()).isEqualTo(2);
+ jobs.remove("jobId1");
+ jobs.remove("jobId2");
+ }
+ {
+ // Restore the jobs, no jobs in database
+ EiJobs jobs = new EiJobs(this.applicationConfig);
+ jobs.restoreJobsFromDatabase();
+ assertThat(jobs.size()).isEqualTo(0);
+ }
+
+ this.eiJobs.remove("jobId1"); // removing a job when the db file is gone
+ assertThat(this.eiJobs.size()).isEqualTo(1);
+ }
+
private void deleteEiProducer(String eiProducerId) {
String url = ProducerConsts.API_ROOT + "/eiproducers/" + eiProducerId;
restClient().deleteForEntity(url).block();