Merge "To optimize the shared module and apply it into the page"
diff --git a/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts b/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts
index b3ed616..98eef9a 100644
--- a/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts
+++ b/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts
@@ -152,7 +152,7 @@
getDbEncryptList(flag): Observable<any> {
return this.http
- .get(prefix + "dbs/list?tool=" + flag)
+ .get(prefix + "dbs/list?isDb=" + flag)
.pipe(retry(1), map(this.extractData), catchError(this.handleError));
}
diff --git a/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts b/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts
index 1acd617..b8dbb0e 100644
--- a/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts
+++ b/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts
@@ -30,11 +30,7 @@
import { ToastrNotificationService } from "src/app/shared/components/toastr-notification/toastr-notification.service";
import {AlertComponent} from "../../../shared/components/alert/alert.component";
import {NewKafkaModalComponent} from "./new-kafka-modal/new-kafka-modal.component";
-import {ModalContentData} from "../../../shared/modules/modal/modal.data";
-import {ModalDemoComponent} from "../../test/modal-demo/modal-demo.component";
-import {ModalComponent} from "../../../shared/modules/modal/modal.component";
import {EditKafkaModalComponent} from "./edit-kafka-modal/edit-kafka-modal.component";
-import {el} from "@angular/platform-browser/testing/src/browser_util";
@Component({
selector: 'app-kafka-list',
@@ -51,6 +47,7 @@
Kafka_New: Kafka;
Kafka_Newbody: Kafka;
cardIconPathList: any = [];
+ kafkaData: any = [];
constructor(
private kafkaApiService: RestApiService,
@@ -74,6 +71,7 @@
if (this.kafkas.length > 0) {
let a = "assets/icons/kafka_able.svg";
let b = "assets/icons/kafka_disable.svg";
+ this.cardIconPathList.splice(0,this.cardIconPathList.length);
for (let i = 0; i < this.kafkas.length; i++) {
this.cardIconPath = (this.kafkas[i].enabled == true) ? a : b;
this.cardIconPathList.push(this.cardIconPath);
@@ -98,6 +96,16 @@
return data;
}
+ async getKafkaDetailModal(id: number) {
+ this.kafkaData = [];
+ this.kafkaData = await this.getKafkaModal(id);
+ return this.kafkaData;
+ }
+
+ getKafkaModal(id: number) {
+ return this.kafkaApiService.getKafka(id).toPromise();
+ }
+
async initKafkasList(kafkaList: []) {
let k: Kafka[] = [];
if (kafkaList.length > 0) {
@@ -208,33 +216,38 @@
}
editKafkaModal(id: number) {
- console.log("id", id)
- const index = this.kafkaList.findIndex(t => t.id === id);
- const modalRef = this.modalService.open(EditKafkaModalComponent, {
- windowClass: "dl-md-modal kafkas",
- centered: true
- });
- modalRef.componentInstance.editKafka = this.kafkaList[index];
- modalRef.componentInstance.passEntry.subscribe(receivedEntry => {
- this.Kafka_New = receivedEntry;
- this.kafkaApiService
- .updateKafka(this.Kafka_New)
- .subscribe(
- res => {
- if (res.statusCode == 200) {
- this.kafkaList[index] = this.Kafka_New;
- this.kafkaList = [...this.kafkaList];
- this.notificationService.success("SUCCESSFULLY_UPDATED");
- } else {
- this.notificationService.error("FAILED_UPDATED");
+ this.getKafkaDetailModal(id).then(data => {
+ console.log("id", id);
+ const index = this.kafkaList.findIndex(t => t.id === id);
+ const modalRef = this.modalService.open(EditKafkaModalComponent, {
+ windowClass: "dl-md-modal kafkas",
+ centered: true
+ });
+ modalRef.componentInstance.editKafka = data;
+ modalRef.componentInstance.passEntry.subscribe(receivedEntry => {
+ this.Kafka_New = receivedEntry;
+ this.kafkaApiService
+ .updateKafka(this.Kafka_New)
+ .subscribe(
+ res => {
+ this.spinner.hide();
+ if (res.statusCode == 200) {
+ this.kafkaList[index] = this.Kafka_New;
+ this.kafkaList = [...this.kafkaList];
+ this.notificationService.success("SUCCESSFULLY_UPDATED");
+ this.initList();
+ } else {
+ this.notificationService.error("FAILED_UPDATED");
+ }
+ modalRef.close();
+ },
+ err => {
+ this.notificationService.error(err);
+ modalRef.close();
}
- modalRef.close();
- },
- err => {
- this.notificationService.error(err);
- modalRef.close();
- }
- );
- })
+ );
+ });
+ });
+
}
}
diff --git a/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts b/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts
index be0d253..65de6ad 100644
--- a/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts
+++ b/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts
@@ -50,7 +50,7 @@
private adminService: AdminService,
private notificationService: ToastrNotificationService,
private modalService: NgbModal,
- private dbApiService: RestApiService,
+ private dbApiService: RestApiService
) {
this.adminService.setTitle("SIDEBAR.DASHBOARDLIST");
this.initData().then(data => { });
diff --git a/components/datalake-handler/feeder/src/assembly/Dockerfile b/components/datalake-handler/feeder/Dockerfile
similarity index 78%
rename from components/datalake-handler/feeder/src/assembly/Dockerfile
rename to components/datalake-handler/feeder/Dockerfile
index 7cd6380..20444f9 100644
--- a/components/datalake-handler/feeder/src/assembly/Dockerfile
+++ b/components/datalake-handler/feeder/Dockerfile
@@ -1,4 +1,4 @@
-FROM openjdk:8-jre
+FROM openjdk:8-jre-slim
MAINTAINER Guobiao Mo <guobiaomo@chinamobile.com>
@@ -12,7 +12,7 @@
WORKDIR /home/datalake
#add the fat jar
-COPY target/feeder-1.0.0-SNAPSHOT.jar /home/datalake/
+COPY target/${JAR_FILE} /home/datalake/
COPY src/assembly/run.sh /home/datalake/
CMD ["sh", "run.sh"]
diff --git a/components/datalake-handler/feeder/pom.xml b/components/datalake-handler/feeder/pom.xml
index 560e1e7..67f62a0 100644
--- a/components/datalake-handler/feeder/pom.xml
+++ b/components/datalake-handler/feeder/pom.xml
@@ -1,6 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -15,108 +14,146 @@
<packaging>jar</packaging>
<name>DataLake Feeder</name>
+ <properties>
+ <swagger.version>2.9.2</swagger.version>
+ <dockerfile-maven.version>1.4.13</dockerfile-maven.version>
+ <docker.image.path>onap/org.onap.dcaegen2.services.datalakefeeder</docker.image.path>
+ </properties>
<dependencies>
<dependency>
+ <groupId>org.jdom</groupId>
+ <artifactId>jdom2</artifactId>
+ <version>2.0.6</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.facebook.presto</groupId>
+ <artifactId>presto-jdbc</artifactId>
+ <version>0.229</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId>
+ <version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
+ <version>20190722</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
+ <version>4.5.10</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
+ <version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
+ <version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
+ <version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
+ <version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-couchbase</artifactId>
+ <version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
+ <version>${springboot.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
+ <version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>${elasticsearchjava.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
+ <version>2.6</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
+ <version>2.8.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
+ <version>1.18.10</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.druid</groupId>
<artifactId>tranquility-core_2.11</artifactId>
+ <version>0.8.3</version>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
+ <version>2.1</version>
</dependency>
@@ -136,26 +173,27 @@
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-validator</artifactId>
- <version>6.0.10.Final</version>
+ <version>6.1.0.Final</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
- <version>2.9.2</version>
+ <version>${swagger.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
- <version>2.9.2</version>
+ <version>${swagger.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
+ <version>${mongojava.version}</version>
</dependency>
<dependency>
<groupId>com.couchbase.mock</groupId>
@@ -165,4 +203,24 @@
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <version>${dockerfile-maven.version}</version>
+ <configuration>
+ <!-- repository>docker.io/moguobiao/datalake-feeder-maven</repository-->
+ <repository>${onap.nexus.dockerregistry.daily}/${docker.image.path}</repository>
+ <tag>${project.version}</tag>
+ <dockerfile>Dockerfile</dockerfile>
+ <!-- useMavenSettingsForAuth>true</useMavenSettingsForAuth-->
+ <buildArgs>
+ <JAR_FILE>${project.build.finalName}.jar</JAR_FILE>
+ </buildArgs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql b/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql
index 8a91427..3f495e2 100644
--- a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql
+++ b/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql
@@ -2,7 +2,7 @@
* ============LICENSE_START=======================================================
* ONAP : DATALAKE
* ================================================================================
-* Copyright 2019 China Mobile
+* Copyright 2019-2020 China Mobile
*=================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -98,7 +98,7 @@
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `topic` (
- `id` int(11) NOT NULL,
+ `id` int(11) NOT NULL AUTO_INCREMENT,
`aggregate_array_path` varchar(255) DEFAULT NULL,
`correlate_cleared_message` bit(1) NOT NULL DEFAULT b'0',
`data_format` varchar(255) DEFAULT NULL,
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java
index 41995e0..9865e0a 100644
--- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java
@@ -121,14 +121,14 @@
Kafka oldKafka = kafkaService.getKafkaById(id);
if (oldKafka == null) {
- sendError(response, 400, "kafka not found "+id);
+ sendError(response, 400, "kafka not found, ID: "+id);
} else {
kafkaRepository.delete(oldKafka);
response.setStatus(204);
}
}
- @GetMapping("")
+ /*@GetMapping("")
@ResponseBody
@ApiOperation(value="List all Kafka id")
public List<Integer> list() {
@@ -139,12 +139,30 @@
retString.add(k.getId());
}
return retString;
- }
+ }*/
+ @GetMapping("")
+ @ResponseBody
+ @ApiOperation(value="List all Kafkas")
public List<KafkaConfig> queryAllKafka(){
return kafkaService.getAllKafka();
}
+ @GetMapping("/{id}")
+ @ResponseBody
+ @ApiOperation(value="Get detail of kafka by id")
+ public KafkaConfig getKafkaDetail(@PathVariable int id, HttpServletResponse response) throws IOException {
+ log.info("Get detail of kafka, ID: " + id);
+ Kafka oldKafka = kafkaService.getKafkaById(id);
+ if (oldKafka == null) {
+ sendError(response, 400, "kafka not found, ID: "+id);
+ return null;
+ } else {
+ log.info("ResponseBody......" + oldKafka.getKafkaConfig());
+ return oldKafka.getKafkaConfig();
+ }
+ }
+
private PostReturnBody<KafkaConfig> mkPostReturnBody(int statusCode, Kafka kafka) {
PostReturnBody<KafkaConfig> retBody = new PostReturnBody<>();
retBody.setStatusCode(statusCode);
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/TopicNameController.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/TopicNameController.java
new file mode 100644
index 0000000..570dcd1
--- /dev/null
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/TopicNameController.java
@@ -0,0 +1,54 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP : DataLake
+ * ================================================================================
+ * Copyright 2019-2020 China Mobile
+ *=================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.datalake.feeder.controller;
+
+import io.swagger.annotations.ApiOperation;
+import org.onap.datalake.feeder.domain.TopicName;
+import org.onap.datalake.feeder.repository.TopicNameRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@RestController
+@RequestMapping(value = "/topicNames", produces = { MediaType.APPLICATION_JSON_VALUE })
+public class TopicNameController {
+
+ @Autowired
+ private TopicNameRepository topicNameRepository;
+
+ @GetMapping("")
+ @ResponseBody
+ @ApiOperation(value="List all topicNames")
+ public List<String> list() {
+ Iterable<TopicName> ret = topicNameRepository.findAll();
+ List<String> retString = new ArrayList<>();
+ for(TopicName item : ret) {
+ retString.add(item.getId());
+ }
+ return retString;
+ }
+}
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java
index 0de004d..fcbe613 100644
--- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java
@@ -2,7 +2,7 @@
* ============LICENSE_START=======================================================
* ONAP : DataLake
* ================================================================================
-* Copyright 2019 China Mobile
+* Copyright 2019-2020 China Mobile
*=================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,6 +23,8 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
import javax.persistence.Column;
import javax.persistence.Entity;
@@ -33,6 +35,8 @@
import javax.persistence.ManyToMany;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
+import javax.persistence.GenerationType;
+import javax.persistence.GeneratedValue;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONObject;
@@ -57,6 +61,7 @@
public class Topic {
@Id
@Column(name = "`id`")
+ @GeneratedValue(strategy = GenerationType.IDENTITY)
private Integer id;
@ManyToOne(fetch = FetchType.EAGER)
@@ -201,16 +206,27 @@
Set<Db> topicDb = getDbs();
List<Integer> dbList = new ArrayList<>();
List<Integer> enabledDbList = new ArrayList<>();
+ List<String> enabledDbList2 = new ArrayList<>();
if (topicDb != null) {
for (Db item : topicDb) {
dbList.add(item.getId());
if(item.isEnabled()) {
enabledDbList.add(item.getId());
+ enabledDbList2.add(item.getDbType().getId());
}
}
}
tConfig.setSinkdbs(dbList);
tConfig.setEnabledSinkdbs(enabledDbList);
+ Map<String,Integer> map = new HashMap<>();
+ for (String string : enabledDbList2) {
+ if(map.containsKey(string)) {
+ map.put(string, map.get(string).intValue()+1);
+ }else {
+ map.put(string, new Integer(1));
+ }
+ }
+ tConfig.setCountsDb(map);
Set<Kafka> topicKafka = getKafkas();
List<Integer> kafkaList = new ArrayList<>();
@@ -220,6 +236,7 @@
}
}
tConfig.setKafkas(kafkaList);
+ tConfig.setCountsKafka(kafkaList.size());
return tConfig;
}
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java
index 1bdad2e..c865ec9 100644
--- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java
@@ -2,7 +2,7 @@
* ============LICENSE_START=======================================================
* ONAP : DataLake
* ================================================================================
- * Copyright 2019 QCT
+ * Copyright 2019-2020 QCT
*=================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@
import lombok.Setter;
import java.util.List;
+import java.util.Map;
/**
* JSON request body for Topic manipulation.
@@ -37,7 +38,7 @@
public class TopicConfig {
- private int id;
+ private Integer id;
private String name;
private String login;
private String password;
@@ -52,7 +53,9 @@
private String aggregateArrayPath;
private String flattenArrayPath;
private List<Integer> kafkas;
-
+ private Map<String,Integer> countsDb;
+ private int countsKafka;
+
@Override
public String toString() {
return String.format("TopicConfig %s(enabled=%s, enabledSinkdbs=%s)", name, enabled, enabledSinkdbs);
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java
index a02cd6a..3bdbcdb 100644
--- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -60,6 +61,9 @@
@Autowired
private KafkaRepository kafkaRepository;
+ @Autowired
+ private TopicNameService topicNameService;
+
//effectiveTopic Map, 1st key is kafkaId, 2nd is topic name, the value is a list of EffectiveTopic.
private Map<Integer, Map<String, List<EffectiveTopic>>> effectiveTopicMap = new HashMap<>();
//private Map<String, TopicConfig> effectiveTopicConfigMap;
@@ -114,7 +118,7 @@
log.info("TopicConfigPollingService started.");
while (active) {
- try { //sleep first since we already pool in init()
+ try { //sleep first since we already called poll() in init()
Thread.sleep(config.getCheckTopicInterval());
if(!active) {
break;
@@ -138,6 +142,7 @@
log.info("activeTopics list is updated, new={}", newTopics);
activeTopicMap.put(kafkaId, newTopics);
+ //update version
currentActiveTopicsVersionMap.put(kafkaId, currentActiveTopicsVersionMap.getOrDefault(kafkaId, 1)+1);
} else {
log.debug("activeTopics list is not updated.");
@@ -156,14 +161,20 @@
}
private Map<Integer, Set<String>> poll() throws IOException {
+ Set<String> allTopicNames = new HashSet<>();
+
Map<Integer, Set<String>> ret = new HashMap<>();
Iterable<Kafka> kafkas = kafkaRepository.findAll();
for (Kafka kafka : kafkas) {
if (kafka.isEnabled()) {
Set<String> topics = poll(kafka);
ret.put(kafka.getId(), topics);
+ allTopicNames.addAll(topics);
}
}
+
+ topicNameService.update(allTopicNames);
+
return ret;
}
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java
new file mode 100644
index 0000000..021d2c9
--- /dev/null
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java
@@ -0,0 +1,59 @@
+/*
+* ============LICENSE_START=======================================================
+* ONAP : DATALAKE
+* ================================================================================
+* Copyright 2019 China Mobile
+*=================================================================================
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+* ============LICENSE_END=========================================================
+*/
+
+package org.onap.datalake.feeder.service;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.onap.datalake.feeder.domain.TopicName;
+import org.onap.datalake.feeder.repository.TopicNameRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Service for TopicName
+ *
+ * @author Guobiao Mo
+ *
+ */
+@Service
+public class TopicNameService {
+ private final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ @Autowired
+ private TopicNameRepository topicNameRepository;
+
+ public void update(Collection<String> allTopicNames) {
+
+ List<TopicName> all = allTopicNames.stream().map(s-> new TopicName(s)).collect(Collectors.toList());
+ List<TopicName> allInDb = (List<TopicName>) topicNameRepository.findAll();
+
+ Collection<TopicName> additions = CollectionUtils.subtract(all, allInDb);
+
+ if(!additions.isEmpty())
+ topicNameRepository.saveAll(additions);
+
+ }
+}
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java
index 2f0761a..c26d980 100644
--- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java
+++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java
@@ -2,7 +2,7 @@
* ============LICENSE_START=======================================================
* ONAP : DATALAKE
* ================================================================================
-* Copyright 2019 China Mobile
+* Copyright 2019-2020 China Mobile
*=================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,6 +34,7 @@
import org.onap.datalake.feeder.domain.EffectiveTopic;
import org.onap.datalake.feeder.domain.Kafka;
import org.onap.datalake.feeder.domain.Topic;
+import org.onap.datalake.feeder.domain.TopicName;
import org.onap.datalake.feeder.repository.DbRepository;
import org.onap.datalake.feeder.repository.KafkaRepository;
import org.onap.datalake.feeder.repository.TopicNameRepository;
@@ -101,6 +102,8 @@
return ret;
}
+ // for unique topic string, one can create multiple 'topic' in admin UI.
+ // for example, one 'topic' setting correlates events, and sends data to ES, another 'topic' sends data to HDFS without such setting
//TODO use query
public List<Topic> findTopics(Kafka kafka, String topicStr) {
List<Topic> ret = new ArrayList<>();
@@ -147,7 +150,10 @@
private void fillTopic(TopicConfig tConfig, Topic topic) {
Set<Db> relateDb = new HashSet<>();
topic.setId(tConfig.getId());
- topic.setTopicName(topicNameRepository.findById(tConfig.getName()).get());
+ Optional<TopicName> t = topicNameRepository.findById(tConfig.getName());
+ if (!t.isPresent())
+ throw new IllegalArgumentException("Can not find topicName in TopicName, topic name " + tConfig.getName());
+ topic.setTopicName(t.get());
topic.setLogin(tConfig.getLogin());
topic.setPass(tConfig.getPassword());
topic.setEnabled(tConfig.isEnabled());
diff --git a/components/pm-subscription-handler/pmsh_service/mod/config_handler.py b/components/pm-subscription-handler/pmsh_service/mod/config_handler.py
new file mode 100755
index 0000000..e9edbca
--- /dev/null
+++ b/components/pm-subscription-handler/pmsh_service/mod/config_handler.py
@@ -0,0 +1,78 @@
+# ============LICENSE_START===================================================
+# Copyright (C) 2019-2020 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=====================================================
+
+import json
+from os import environ
+
+import requests
+from tenacity import retry, wait_fixed, stop_after_attempt
+
+from pmsh_service.mod import pmsh_logging as logger
+
+
+class ConfigHandler:
+ """ Handles retrieval of PMSH's configuration from Configbinding service."""
+
+ def __init__(self):
+ self.cbs_url = f'http://{self.cbs_hostname}:{str(self.cbs_port)}/' \
+ f'service_component_all/{self.hostname}'
+ self._config = None
+
+ @property
+ def cbs_hostname(self):
+ return _get_environment_variable('CONFIG_BINDING_SERVICE_SERVICE_HOST')
+
+ @property
+ def cbs_port(self):
+ return _get_environment_variable('CONFIG_BINDING_SERVICE_SERVICE_PORT')
+
+ @property
+ def hostname(self):
+ return _get_environment_variable('HOSTNAME')
+
+ @retry(wait=wait_fixed(2), stop=stop_after_attempt(5))
+ def get_config(self):
+ """ Retrieves PMSH's configuration from Configbinding service. If a non-2xx response
+ is received, it retries after 2 seconds for 5 times before raising an exception.
+
+ Returns:
+ dict: Dictionary representation of the the service configuration
+
+ Raises:
+ Exception: If any error occurred pulling configuration from Configbinding service.
+ """
+ if self._config is None:
+ logger.debug('No configuration found, pulling from Configbinding Service.')
+ try:
+ response = requests.get(self.cbs_url)
+ response.raise_for_status()
+ self._config = response.json()
+ logger.debug(f'PMSH Configuration from Configbinding Service: {self._config}')
+ return json.loads(self._config)
+ except Exception as err:
+ raise Exception(f'Error retrieving configuration from CBS: {err}')
+ else:
+ return self._config
+
+
+def _get_environment_variable(env_var_key):
+ try:
+ env_var = environ[env_var_key]
+ except KeyError as error:
+ raise KeyError(f'Environment variable {env_var_key} must be set. {error}')
+ return env_var
diff --git a/components/pm-subscription-handler/pmsh_service/pmsh_service.py b/components/pm-subscription-handler/pmsh_service/pmsh_service.py
index 6764c75..8832f57 100755
--- a/components/pm-subscription-handler/pmsh_service/pmsh_service.py
+++ b/components/pm-subscription-handler/pmsh_service/pmsh_service.py
@@ -1,5 +1,5 @@
# ============LICENSE_START===================================================
-# Copyright (C) 2019 Nordix Foundation.
+# Copyright (C) 2019-2020 Nordix Foundation.
# ============================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/components/pm-subscription-handler/setup.py b/components/pm-subscription-handler/setup.py
index 9012a04..a4d9ada 100644
--- a/components/pm-subscription-handler/setup.py
+++ b/components/pm-subscription-handler/setup.py
@@ -30,6 +30,5 @@
python_requires='>=3',
install_requires=[
"requests==2.22.0",
- "aiohttp==3.6.2",
- "onappylog==1.0.9"],
+ "tenacity==6.0.0"],
)
diff --git a/components/pm-subscription-handler/tests/config_handler_test.py b/components/pm-subscription-handler/tests/config_handler_test.py
new file mode 100755
index 0000000..fcc25d6
--- /dev/null
+++ b/components/pm-subscription-handler/tests/config_handler_test.py
@@ -0,0 +1,112 @@
+# ============LICENSE_START===================================================
+# Copyright (C) 2019-2020 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=====================================================
+
+import json
+import unittest
+from os import environ
+from os import path
+from unittest.mock import patch
+
+import requests
+import responses
+from tenacity import wait_none
+
+from pmsh_service.mod.config_handler import ConfigHandler
+
+
+class ConfigHandlerTest(unittest.TestCase):
+
+ def setUp(self):
+ self.env_vars = {'CONFIG_BINDING_SERVICE_SERVICE_HOST': 'cbs_hostname',
+ 'CONFIG_BINDING_SERVICE_SERVICE_PORT': '10000',
+ 'HOSTNAME': 'hostname'}
+ for key, value in self.env_vars.items():
+ environ[key] = value
+ self.cbs_url = 'http://cbs_hostname:10000/service_component_all/hostname'
+ self.expected_config = self._get_expected_config()
+
+ def test_missing_environment_variable(self):
+ for key, value in self.env_vars.items():
+ with self.assertRaises(KeyError):
+ environ.pop(key)
+ test_value = globals()[value]
+ test_value()
+ environ[key] = value
+
+ @responses.activate
+ def test_get_config_success(self):
+ responses.add(responses.GET, self.cbs_url, json=json.dumps(self.expected_config),
+ status=200)
+
+ config_handler = ConfigHandler()
+ config_handler.get_config.retry.wait = wait_none()
+
+ self.assertEqual(self.expected_config, config_handler.get_config())
+
+ def test_get_config_already_exists(self):
+ config_handler = ConfigHandler()
+ expected_config = self._get_expected_config()
+ config_handler._config = expected_config
+
+ with patch.object(requests, 'get') as mock_get_request:
+ actual_config = config_handler.get_config()
+
+ self.assertEqual(0, mock_get_request.call_count)
+ self.assertEqual(expected_config, actual_config)
+
+ @responses.activate
+ def test_get_config_error(self):
+ responses.add(responses.GET, self.cbs_url, status=404)
+ config_handler = ConfigHandler()
+ config_handler.get_config.retry.wait = wait_none()
+
+ with self.assertRaises(Exception):
+ config_handler.get_config()
+
+ @responses.activate
+ def test_get_config_max_retries_error(self):
+ retry_limit = 5
+ config_handler = ConfigHandler()
+ config_handler.get_config.retry.wait = wait_none()
+
+ for __ in range(retry_limit):
+ responses.add(responses.GET, self.cbs_url, status=500)
+
+ with self.assertRaises(Exception):
+ config_handler.get_config()
+ self.assertEqual(retry_limit, len(responses.calls))
+
+ @responses.activate
+ def test_get_config_less_than_5_retries_success(self):
+ retry_attempts = 4
+ responses.add(responses.GET, self.cbs_url, status=500)
+ responses.add(responses.GET, self.cbs_url, status=400)
+ responses.add(responses.GET, self.cbs_url, status=300)
+ responses.add(responses.GET, self.cbs_url, json=json.dumps(self.expected_config),
+ status=200)
+
+ config_handler = ConfigHandler()
+ config_handler.get_config.retry.wait = wait_none()
+ config_handler.get_config()
+
+ self.assertEqual(retry_attempts, len(responses.calls))
+
+ @staticmethod
+ def _get_expected_config():
+ with open(path.join(path.dirname(__file__), 'expected_config.json'))as json_file:
+ return json.load(json_file)
diff --git a/components/pm-subscription-handler/tests/expected_config.json b/components/pm-subscription-handler/tests/expected_config.json
new file mode 100755
index 0000000..43f67e8
--- /dev/null
+++ b/components/pm-subscription-handler/tests/expected_config.json
@@ -0,0 +1,68 @@
+{
+ "config":{},
+ "policy": {
+ "subscription": {
+ "subscriptionName": "someExtraPM-AllKista-gNB-R2B",
+ "administrativeState": "UNLOCKED",
+ "fileBasedGP": 15,
+ "fileLocation": "c:\/\/PM",
+ "nfTypeModelInvariantId": "2829292",
+ "nfFilter": {
+ "swVersions": [
+ "A21",
+ "B"
+ ],
+ "nfNames": [
+ "ABC",
+ "DEF",
+ "foo.*"
+ ]
+ },
+ "measurementGroups": [
+ {
+ "measurementGroup": {
+ "measurementTypes": [
+ {
+ "measurementType": "countera"
+ },
+ {
+ "measurementType": "counterb"
+ }
+ ],
+ "managedObjectDNsBasic": [
+ {
+ "DN": "dna"
+ },
+ {
+ "DN": "dnb"
+ }
+ ]
+ }
+ },
+ {
+ "measurementGroup": {
+ "measurementTypes": [
+ {
+ "measurementType": "counterc"
+ },
+ {
+ "measurementType": "counterd"
+ }
+ ],
+ "managedObjectDNsBasic": [
+ {
+ "DN": "dnc"
+ },
+ {
+ "DN": "dnd"
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
+
+
+
+}
\ No newline at end of file
diff --git a/components/pm-subscription-handler/tox.ini b/components/pm-subscription-handler/tox.ini
index fcbb647..523338d 100644
--- a/components/pm-subscription-handler/tox.ini
+++ b/components/pm-subscription-handler/tox.ini
@@ -26,6 +26,7 @@
pytest
coverage
pytest-cov
+ responses==0.10.7
setenv =
PYTHONPATH={toxinidir}/pmsh_service:{toxinidir}/pmsh_service/mod:{toxinidir}/tests
commands=
@@ -39,3 +40,4 @@
[flake8]
max-line-length=100
+ignore = E999