Add HTTPS to collect files from xNFs
- plus small refactoring related to above
- update to version 1.5.3
Issue-ID: DCAEGEN2-2528
Signed-off-by: Krzysztof Gajewski <krzysztof.gajewski@nokia.com>
Change-Id: I2531c85967964f1359bafd5b694afbf662edf54e
diff --git a/Changelog.md b/Changelog.md
index 7fe3275..7eeed1c 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,15 @@
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
+## [1.5.3] - 11/02/2020
+### Added
+- HTTPS support for DFC
+- test related to HTTPS support
+- counters for http(s)
+### Changed
+- ftp clients and tests refactoring
+- app config related to certificates unified for ftpes and https
+
## [1.5.1] - 04/01/2020
### Added
- HTTP support for DFC
diff --git a/datafile-app-server/dpo/spec/datafile-component-spec.json b/datafile-app-server/dpo/spec/datafile-component-spec.json
index a650518..9de6913 100644
--- a/datafile-app-server/dpo/spec/datafile-component-spec.json
+++ b/datafile-app-server/dpo/spec/datafile-component-spec.json
@@ -61,7 +61,7 @@
],
"parameters": [
{
- "name": "dmaap.ftpesConfig.keyCert",
+ "name": "dmaap.certificateConfig.keyCert",
"value": "/opt/app/datafile/etc/cert/cert.p12",
"description": "",
"designer_editable": true,
@@ -71,7 +71,7 @@
"required": true
},
{
- "name": "dmaap.ftpesConfig.keyPasswordPath",
+ "name": "dmaap.certificateConfig.keyPasswordPath",
"value": "/opt/app/datafile/etc/cert/p12.pass",
"description": "",
"designer_editable": true,
@@ -81,7 +81,7 @@
"required": true
},
{
- "name": "dmaap.ftpesConfig.trustedCa",
+ "name": "dmaap.certificateConfig.trustedCa",
"value": "/opt/app/datafile/etc/cert/trust.jks",
"description": "",
"designer_editable": true,
@@ -91,7 +91,7 @@
"required": true
},
{
- "name": "dmaap.ftpesConfig.trustedCaPasswordPath",
+ "name": "dmaap.certificateConfig.trustedCaPasswordPath",
"value": "/opt/app/datafile/etc/cert/trust.pass",
"description": "",
"designer_editable": true,
diff --git a/datafile-app-server/pom.xml b/datafile-app-server/pom.xml
index 353b375..5e0330b 100644
--- a/datafile-app-server/pom.xml
+++ b/datafile-app-server/pom.xml
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ ============LICENSE_START=======================================================
- ~ Copyright (C) 2018-2020 NOKIA Intellectual Property. All rights reserved.
+ ~ Copyright (C) 2018-2021 NOKIA Intellectual Property. All rights reserved.
~ Copyright (C) 2018-2021 Nordix Foundation. All rights reserved.
~ Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
~ ================================================================================
@@ -26,7 +26,7 @@
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.3-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
@@ -158,6 +158,16 @@
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <scope>test</scope>
+ </dependency>
<!--REQUIRED TO GENERATE DOCUMENTATION -->
<dependency>
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java
index afa3aae..b9aa644 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START=======================================================
* Copyright (C) 2019 Nordix Foundation. All rights reserved.
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,10 +28,10 @@
*
*/
public enum Scheme {
- FTPES, SFTP, HTTP;
+ FTPES, SFTP, HTTP, HTTPS;
public static final String DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG = "DFC does not support protocol ";
- public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP and HTTP";
+ public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = ". Supported protocols are FTPeS, sFTP, HTTP and HTTPS";
/**
* Get a <code>Scheme</code> from a string.
@@ -48,10 +48,22 @@
result = Scheme.SFTP;
} else if ("HTTP".equalsIgnoreCase(schemeString)) {
result = Scheme.HTTP;
+ } else if ("HTTPS".equalsIgnoreCase(schemeString)) {
+ result = Scheme.HTTPS;
} else {
throw new DatafileTaskException(
DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + schemeString + SUPPORTED_PROTOCOLS_ERROR_MESSAGE);
}
return result;
}
+
+ /**
+ * Check if <code>Scheme</code> is FTP type or HTTP type.
+ *
+ * @param scheme the <code>Scheme</code> which has to be checked.
+ * @return true if <code>Scheme</code> is FTP type or false if it is HTTP type
+ */
+ public static boolean isFtpScheme(Scheme scheme) {
+ return scheme == SFTP || scheme == FTPES;
+ }
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java
new file mode 100644
index 0000000..79db32d
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java
@@ -0,0 +1,50 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.commons;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * Utility class containing functions used for certificates configuration
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+public final class SecurityUtil {
+ private SecurityUtil() {}
+ private static final Logger logger = LoggerFactory.getLogger(SecurityUtil.class);
+
+ public static String getKeystorePasswordFromFile(String passwordPath) {
+ return getPasswordFromFile(passwordPath, "Keystore");
+ }
+
+ public static String getTruststorePasswordFromFile(String passwordPath) {
+ return getPasswordFromFile(passwordPath, "Truststore");
+ }
+
+ public static String getPasswordFromFile(String passwordPath, String element) {
+ try {
+ return new String(Files.readAllBytes(Paths.get(passwordPath)));
+ } catch (IOException e) {
+ logger.error("{} password file at path: {} cannot be opened ", element, passwordPath);
+ }
+ return "";
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java
index d933e33..b381c02 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation.
+ * All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -37,6 +38,7 @@
import javax.validation.constraints.NotNull;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil;
import org.onap.dcaegen2.collectors.datafile.model.logging.MappedDiagnosticContext;
import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClient;
import org.onap.dcaegen2.services.sdk.rest.services.cbs.client.api.CbsClientFactory;
@@ -76,7 +78,7 @@
Properties systemEnvironment;
private ConsumerConfiguration dmaapConsumerConfiguration;
private Map<String, PublisherConfiguration> publishingConfigurations;
- private FtpesConfig ftpesConfiguration;
+ private CertificateConfig certificateConfiguration;
private SftpConfig sftpConfiguration;
private Disposable refreshConfigTask = null;
@@ -163,8 +165,8 @@
return cfg;
}
- public synchronized FtpesConfig getFtpesConfiguration() {
- return ftpesConfiguration;
+ public synchronized CertificateConfig getCertificateConfiguration() {
+ return certificateConfiguration;
}
public synchronized SftpConfig getSftpConfiguration() {
@@ -193,7 +195,7 @@
CloudConfigParser parser =
new CloudConfigParser(configurationObject, systemEnvironment);
setConfiguration(parser.getConsumerConfiguration(),
- parser.getDmaapPublisherConfigurations(), parser.getFtpesConfig(),
+ parser.getDmaapPublisherConfigurations(), parser.getCertificateConfig(),
parser.getSftpConfig());
logConfig();
} catch (DatafileTaskException e) {
@@ -204,7 +206,7 @@
private void logConfig() {
logger.debug("Read and parsed sFTP configuration: [{}]", sftpConfiguration);
- logger.debug("Read and parsed FTPes configuration: [{}]", ftpesConfiguration);
+ logger.debug("Read and parsed FTPes / HTTPS configuration: [{}]", certificateConfiguration);
logger.debug("Read and parsed DMaaP configuration: [{}]", dmaapConsumerConfiguration);
logger.debug("Read and parsed Publish configuration: [{}]", publishingConfigurations);
}
@@ -226,12 +228,14 @@
}
private synchronized void setConfiguration(@NotNull ConsumerConfiguration consumerConfiguration,
- @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull FtpesConfig ftpesConfig,
- @NotNull SftpConfig sftpConfig) {
+ @NotNull Map<String, PublisherConfiguration> publisherConfiguration, @NotNull CertificateConfig certificateConfig,
+ @NotNull SftpConfig sftpConfig) throws DatafileTaskException {
this.dmaapConsumerConfiguration = consumerConfiguration;
this.publishingConfigurations = publisherConfiguration;
- this.ftpesConfiguration = ftpesConfig;
+ this.certificateConfiguration = certificateConfig;
this.sftpConfiguration = sftpConfig;
+ HttpsClientConnectionManagerUtil.setupOrUpdate(certificateConfig.keyCert(), certificateConfig.keyPasswordPath(),
+ certificateConfig.trustedCa(), certificateConfig.trustedCaPasswordPath());
}
JsonElement getJsonElement(InputStream inputStream) {
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java
similarity index 95%
rename from datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java
rename to datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java
index 9e9da7d..1d8b614 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/FtpesConfig.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java
@@ -30,7 +30,7 @@
@Value.Immutable
@Value.Style(builder = "new", redactedMask = "####")
@Gson.TypeAdapters
-public abstract class FtpesConfig implements Serializable {
+public abstract class CertificateConfig implements Serializable {
private static final long serialVersionUID = 1L;
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java
index 6ace4aa..d6b8643 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CloudConfigParser.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START=======================================================
- * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation.
+ * All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -187,12 +188,12 @@
* @return the xNF communication security configuration.
* @throws DatafileTaskException if a member of the configuration is missing.
*/
- public @NotNull FtpesConfig getFtpesConfig() throws DatafileTaskException {
- return new ImmutableFtpesConfig.Builder() //
- .keyCert(getAsString(jsonObject, "dmaap.ftpesConfig.keyCert"))
- .keyPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.keyPasswordPath"))
- .trustedCa(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCa"))
- .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.ftpesConfig.trustedCaPasswordPath")) //
+ public @NotNull CertificateConfig getCertificateConfig() throws DatafileTaskException {
+ return new ImmutableCertificateConfig.Builder() //
+ .keyCert(getAsString(jsonObject, "dmaap.certificateConfig.keyCert"))
+ .keyPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.keyPasswordPath"))
+ .trustedCa(getAsString(jsonObject, "dmaap.certificateConfig.trustedCa"))
+ .trustedCaPasswordPath(getAsString(jsonObject, "dmaap.certificateConfig.trustedCaPasswordPath")) //
.build();
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java
index 9bacec8..a82b547 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START======================================================================
* Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -22,9 +22,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
@@ -43,6 +41,7 @@
import org.apache.commons.net.ftp.FTPSClient;
import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.slf4j.Logger;
@@ -194,13 +193,7 @@
protected TrustManager getTrustManager(Path trustedCaPath, String trustedCaPasswordPath)
throws KeyStoreException, NoSuchAlgorithmException, IOException, CertificateException {
- String trustedCaPassword = "";
- try {
- trustedCaPassword = new String(Files.readAllBytes(Paths.get(trustedCaPasswordPath)));
- } catch (IOException e) {
- logger.error("Truststore password file at path: {} cannot be opened ", trustedCaPasswordPath);
- e.printStackTrace();
- }
+ String trustedCaPassword = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath);
synchronized (FtpesClient.class) {
if (theTrustManager == null) {
theTrustManager = createTrustManager(trustedCaPath, trustedCaPassword);
@@ -211,14 +204,7 @@
protected KeyManager getKeyManager(Path keyCertPath, String keyCertPasswordPath)
throws IOException, GeneralSecurityException {
- String keyCertPassword = "";
- try {
- keyCertPassword = new String(Files.readAllBytes(Paths.get(keyCertPasswordPath)));
- } catch (IOException e) {
- logger.error("Keystore password file at path: {} cannot be opened ", keyCertPasswordPath);
- e.printStackTrace();
- }
-
+ String keyCertPassword = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath);
synchronized (FtpesClient.class) {
if (theKeyManager == null) {
theKeyManager = createKeyManager(keyCertPath, keyCertPassword);
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
index 86bfc21..3ccc9fb 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
@@ -1,6 +1,6 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -37,6 +37,11 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+/**
+ * Gets file from PNF with HTTP protocol.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
public class DfcHttpClient implements FileCollectClient {
//Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS
@@ -111,7 +116,7 @@
try {
long numBytes = Files.copy(response, localFile);
logger.trace("Transmission was successful - {} bytes downloaded.", numBytes);
- logger.trace("CollectFile fetched: {}", localFile.toString());
+ logger.trace("CollectFile fetched: {}", localFile);
response.close();
} catch (IOException e) {
errorMessages.set(new Exception("Error fetching file with", e));
@@ -139,7 +144,7 @@
}
@NotNull protected String prepareUri(String remoteFile) {
- int port = fileServerData.port().isPresent() ? fileServerData.port().get() : HttpUtils.HTTP_DEFAULT_PORT;
+ int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT);
return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile;
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
new file mode 100644
index 0000000..3090815
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
@@ -0,0 +1,170 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.conn.HttpHostConnectException;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
+import org.jetbrains.annotations.NotNull;
+import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
+import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.service.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLHandshakeException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+/**
+ * Gets file from PNF with HTTPS protocol.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+public class DfcHttpsClient implements FileCollectClient {
+
+ protected CloseableHttpClient httpsClient;
+
+ private static final Logger logger = LoggerFactory.getLogger(DfcHttpsClient.class);
+ private static final int FIFTEEN_SECONDS = 15 * 1000;
+
+ private final FileServerData fileServerData;
+ private final PoolingHttpClientConnectionManager connectionManager;
+
+ public DfcHttpsClient(FileServerData fileServerData, PoolingHttpClientConnectionManager connectionManager) {
+ this.fileServerData = fileServerData;
+ this.connectionManager = connectionManager;
+ }
+
+ @Override public void open() {
+ logger.trace("Setting httpsClient for file download.");
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(true)
+ .build();
+
+ RequestConfig requestConfig = RequestConfig.custom()
+ .setConnectTimeout(FIFTEEN_SECONDS)
+ .build();
+
+ httpsClient = HttpClients.custom()
+ .setConnectionManager(connectionManager)
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(requestConfig)
+ .build();
+
+ logger.trace("httpsClient prepared for connection.");
+ }
+
+ @Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException {
+ logger.trace("Prepare to collectFile {}", localFile);
+ HttpGet httpGet = new HttpGet(prepareUri(remoteFile));
+ if (basicAuthValidNotPresentOrThrow()) {
+ httpGet.addHeader("Authorization",
+ HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password()));
+ }
+ try {
+ HttpResponse httpResponse = makeCall(httpGet);
+ processResponse(httpResponse, localFile);
+ } catch (IOException e) {
+ throw new DatafileTaskException("Error downloading file from server. ", e);
+ }
+ logger.trace("HTTPS collectFile OK");
+ }
+
+ protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException {
+ if (isAuthDataEmpty()) {
+ return false;
+ }
+ if (isAuthDataFilled()) {
+ return true;
+ }
+ throw new DatafileTaskException("Not sufficient basic auth data for file.");
+ }
+
+ private boolean isAuthDataEmpty() {
+ return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty();
+ }
+
+ private boolean isAuthDataFilled() {
+ return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty();
+ }
+
+ @NotNull protected String prepareUri(String remoteFile) {
+ int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT);
+ return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile;
+ }
+
+ protected HttpResponse makeCall(HttpGet httpGet)
+ throws IOException, DatafileTaskException {
+ try {
+ HttpResponse httpResponse = executeHttpClient(httpGet);
+ if (isResponseOk(httpResponse)) {
+ return httpResponse;
+ }
+
+ EntityUtils.consume(httpResponse.getEntity());
+ throw new NonRetryableDatafileTaskException(
+ "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode()
+ + ". No retry attempts will be done.");
+
+ } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) {
+ throw new NonRetryableDatafileTaskException(
+ "Unable to get file from xNF. No retry attempts will be done.", e);
+ }
+ }
+
+ protected CloseableHttpResponse executeHttpClient(HttpGet httpGet)
+ throws IOException {
+ return httpsClient.execute(httpGet);
+ }
+
+ protected boolean isResponseOk(HttpResponse response) {
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+
+ protected void processResponse(HttpResponse response, Path localFile) throws IOException {
+ logger.trace("Starting to process response.");
+ HttpEntity entity = response.getEntity();
+ InputStream stream = entity.getContent();
+ long numBytes = writeFile(localFile, stream);
+ stream.close();
+ EntityUtils.consume(entity);
+ logger.trace("Transmission was successful - {} bytes downloaded.", numBytes);
+ }
+
+ protected long writeFile(Path localFile, InputStream stream) throws IOException {
+ return Files.copy(stream, localFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ @Override public void close() {
+ logger.trace("Https client has ended downloading process.");
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java
new file mode 100644
index 0000000..e60ec0f
--- /dev/null
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java
@@ -0,0 +1,132 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.FileSystemResource;
+
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+/**
+ * Utility class supplying connection manager for HTTPS protocol.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+public class HttpsClientConnectionManagerUtil {
+
+ private HttpsClientConnectionManagerUtil() {
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(HttpsClientConnectionManagerUtil.class);
+ //Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS
+ private static final int MAX_NUMBER_OF_CONNECTIONS = 200;
+ private static PoolingHttpClientConnectionManager connectionManager;
+
+ public static PoolingHttpClientConnectionManager instance() throws DatafileTaskException {
+ if (connectionManager == null) {
+ throw new DatafileTaskException("ConnectionManager has to be set or update first");
+ }
+ return connectionManager;
+ }
+
+ public static void setupOrUpdate(String keyCertPath, String keyCertPasswordPath, String trustedCaPath,
+ String trustedCaPasswordPath) throws DatafileTaskException {
+ synchronized (HttpsClientConnectionManagerUtil.class) {
+ if (connectionManager != null) {
+ connectionManager.close();
+ connectionManager = null;
+ }
+ setup(keyCertPath, keyCertPasswordPath, trustedCaPath, trustedCaPasswordPath);
+ }
+ logger.trace("HttpsConnectionManager setup or updated");
+ }
+
+ private static void setup(String keyCertPath, String keyCertPasswordPath, String trustedCaPath,
+ String trustedCaPasswordPath) throws DatafileTaskException {
+ try {
+ SSLContextBuilder sslBuilder = SSLContexts.custom();
+ sslBuilder = supplyKeyInfo(keyCertPath, keyCertPasswordPath, sslBuilder);
+ sslBuilder = supplyTrustInfo(trustedCaPath, trustedCaPasswordPath, sslBuilder);
+
+ SSLContext sslContext = sslBuilder.build();
+
+ SSLConnectionSocketFactory sslConnectionSocketFactory =
+ new SSLConnectionSocketFactory(sslContext, new String[] {"TLSv1.2"}, null,
+ (hostname, session) -> true);
+
+ Registry<ConnectionSocketFactory> socketFactoryRegistry =
+ RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslConnectionSocketFactory)
+ .build();
+
+ connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connectionManager.setMaxTotal(MAX_NUMBER_OF_CONNECTIONS);
+
+ } catch (Exception e) {
+ throw new DatafileTaskException("Unable to prepare HttpsConnectionManager : ", e);
+ }
+ }
+
+ private static SSLContextBuilder supplyKeyInfo(String keyCertPath, String keyCertPasswordPath,
+ SSLContextBuilder sslBuilder)
+ throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException,
+ UnrecoverableKeyException {
+ String keyPass = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath);
+ KeyStore keyFile = createKeyStore(keyCertPath, keyPass);
+ return sslBuilder.loadKeyMaterial(keyFile, keyPass.toCharArray());
+ }
+
+ private static KeyStore createKeyStore(String trustedCaPath, String trustedCaPassword)
+ throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException {
+ logger.trace("Creating trust manager from file: {}", trustedCaPath);
+ try (InputStream fis = createInputStream(trustedCaPath)) {
+ KeyStore keyStore = KeyStore.getInstance("PKCS12");
+ keyStore.load(fis, trustedCaPassword.toCharArray());
+ return keyStore;
+ }
+ }
+
+ private static InputStream createInputStream(String localFileName) throws IOException {
+ FileSystemResource realResource = new FileSystemResource(Paths.get(localFileName));
+ return realResource.getInputStream();
+ }
+
+ private static SSLContextBuilder supplyTrustInfo(String trustedCaPath, String trustedCaPasswordPath,
+ SSLContextBuilder sslBuilder)
+ throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
+ String trustPass = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath);
+ File trustStoreFile = new File(trustedCaPath);
+ return sslBuilder.loadTrustMaterial(trustStoreFile, trustPass.toCharArray());
+ }
+}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java
index 8e8d847..d5587e9 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java
@@ -33,7 +33,9 @@
private final AtomicInteger numberOfSubscriptions = new AtomicInteger();
private int noOfCollectedFiles = 0;
private int noOfFailedFtpAttempts = 0;
+ private int noOfFailedHttpAttempts = 0;
private int noOfFailedFtp = 0;
+ private int noOfFailedHttp = 0;
private int noOfFailedPublishAttempts = 0;
private int totalPublishedFiles = 0;
private int noOfFailedPublish = 0;
@@ -62,10 +64,18 @@
noOfFailedFtpAttempts++;
}
+ public synchronized void incNoOfFailedHttpAttempts() {
+ noOfFailedHttpAttempts++;
+ }
+
public synchronized void incNoOfFailedFtp() {
noOfFailedFtp++;
}
+ public synchronized void incNoOfFailedHttp() {
+ noOfFailedHttp++;
+ }
+
public synchronized void incNoOfFailedPublishAttempts() {
noOfFailedPublishAttempts++;
}
@@ -89,7 +99,9 @@
str.append("\n");
str.append(format("collectedFiles", noOfCollectedFiles));
str.append(format("failedFtpAttempts", noOfFailedFtpAttempts));
+ str.append(format("failedHttpAttempts", noOfFailedHttpAttempts));
str.append(format("failedFtp", noOfFailedFtp));
+ str.append(format("failedHttp", noOfFailedHttp));
str.append("\n");
str.append(format("totalPublishedFiles", totalPublishedFiles));
str.append(format("lastPublishedTime", lastPublishedTime));
@@ -113,10 +125,18 @@
return noOfFailedFtpAttempts;
}
+ public int getNoOfFailedHttpAttempts() {
+ return noOfFailedHttpAttempts;
+ }
+
public int getNoOfFailedFtp() {
return noOfFailedFtp;
}
+ public int getNoOfFailedHttp() {
+ return noOfFailedHttp;
+ }
+
public int getNoOfFailedPublishAttempts() {
return noOfFailedPublishAttempts;
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
index 1dca005..e2c1e2f 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
@@ -26,6 +26,7 @@
public final class HttpUtils implements HttpStatus {
public static final int HTTP_DEFAULT_PORT = 80;
+ public static final int HTTPS_DEFAULT_PORT = 443;
private HttpUtils() {
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
index e76d415..cfc7754 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
@@ -23,8 +23,9 @@
import java.util.Map;
import java.util.Optional;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
-import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
@@ -32,6 +33,8 @@
import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
import org.onap.dcaegen2.collectors.datafile.ftp.SftpClientSettings;
import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient;
+import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient;
+import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil;
import org.onap.dcaegen2.collectors.datafile.model.Counters;
import org.onap.dcaegen2.collectors.datafile.model.FileData;
import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
@@ -109,11 +112,11 @@
return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context)));
} catch (NonRetryableDatafileTaskException nre) {
logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre);
- counters.incNoOfFailedFtpAttempts();
+ incFailedAttemptsCounter(fileData);
return Mono.just(Optional.empty()); // Give up
} catch (DatafileTaskException e) {
logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), e);
- counters.incNoOfFailedFtpAttempts();
+ incFailedAttemptsCounter(fileData);
return Mono.error(e);
} catch (Exception throwable) {
logger.warn("Failed to close client: {} {}, reason: {}", fileData.sourceName(), fileData.name(),
@@ -122,6 +125,14 @@
}
}
+ private void incFailedAttemptsCounter(FileData fileData) {
+ if (Scheme.isFtpScheme(fileData.scheme())) {
+ counters.incNoOfFailedFtpAttempts();
+ } else {
+ counters.incNoOfFailedHttpAttempts();
+ }
+ }
+
private FileCollectClient createClient(FileData fileData) throws DatafileTaskException {
switch (fileData.scheme()) {
case SFTP:
@@ -130,6 +141,8 @@
return createFtpesClient(fileData);
case HTTP:
return createHttpClient(fileData);
+ case HTTPS:
+ return createHttpsClient(fileData);
default:
throw new DatafileTaskException("Unhandled protocol: " + fileData.scheme());
}
@@ -163,7 +176,7 @@
}
protected FtpesClient createFtpesClient(FileData fileData) {
- FtpesConfig config = datafileAppConfig.getFtpesConfiguration();
+ CertificateConfig config = datafileAppConfig.getCertificateConfiguration();
return new FtpesClient(fileData.fileServerData(), Paths.get(config.keyCert()), config.keyPasswordPath(),
Paths.get(config.trustedCa()), config.trustedCaPasswordPath());
}
@@ -171,4 +184,8 @@
protected FileCollectClient createHttpClient(FileData fileData) {
return new DfcHttpClient(fileData.fileServerData());
}
+
+ protected FileCollectClient createHttpsClient(FileData fileData) throws DatafileTaskException {
+ return new DfcHttpsClient(fileData.fileServerData(), HttpsClientConnectionManagerUtil.instance());
+ }
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java
index eba0a6c..fa1757e 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasks.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.model.Counters;
@@ -257,7 +258,11 @@
deleteFile(localFilePath, fileData.context);
publishedFilesCache.remove(localFilePath);
currentNumberOfTasks.decrementAndGet();
- counters.incNoOfFailedFtp();
+ if (Scheme.isFtpScheme(fileData.fileData.scheme())) {
+ counters.incNoOfFailedFtp();
+ } else {
+ counters.incNoOfFailedHttp();
+ }
return Mono.empty();
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java
index 7f0c642..bcbe7f8 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfigTest.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2018, 2020-2021 NOKIA Intellectual Property, 2018-2019 Nordix Foundation.
+ * All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -83,12 +84,12 @@
.password("izBJD8nLjawq0HMG") //
.build();
- private static final ImmutableFtpesConfig CORRECT_FTPES_CONFIGURATION = //
- new ImmutableFtpesConfig.Builder() //
+ private static final ImmutableCertificateConfig CORRECT_CERTIFICATE_CONFIGURATION = //
+ new ImmutableCertificateConfig.Builder() //
.keyCert("/src/test/resources/dfc.jks") //
.keyPasswordPath("/src/test/resources/dfc.jks.pass") //
- .trustedCa("/src/test/resources/ftp.jks") //
- .trustedCaPasswordPath("/src/test/resources/ftp.jks.pass") //
+ .trustedCa("/src/test/resources/cert.jks") //
+ .trustedCaPasswordPath("/src/test/resources/cert.jks.pass") //
.build();
private AppConfig appConfigUnderTest;
@@ -120,9 +121,9 @@
Assertions.assertNotNull(publisherCfg);
assertThat(publisherCfg).isEqualToComparingFieldByField(CORRECT_PUBLISHER_CONFIG);
- FtpesConfig ftpesConfig = appConfigUnderTest.getFtpesConfiguration();
- assertThat(ftpesConfig).isNotNull();
- assertThat(ftpesConfig).isEqualToComparingFieldByField(CORRECT_FTPES_CONFIGURATION);
+ CertificateConfig certificateConfig = appConfigUnderTest.getCertificateConfiguration();
+ assertThat(certificateConfig).isNotNull();
+ assertThat(certificateConfig).isEqualToComparingFieldByField(CORRECT_CERTIFICATE_CONFIGURATION);
}
@Test
@@ -157,7 +158,7 @@
assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER))
.hasMessageContaining("No PublishingConfiguration loaded, changeIdentifier: PM_MEAS_FILES");
- Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration());
+ Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration());
}
@Test
@@ -172,7 +173,7 @@
Assertions.assertNull(appConfigUnderTest.getDmaapConsumerConfiguration());
assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER))
.hasMessageContaining(CHANGE_IDENTIFIER);
- Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration());
+ Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration());
}
@Test
@@ -190,7 +191,7 @@
Assertions.assertNull(appConfigUnderTest.getDmaapConsumerConfiguration());
assertThatThrownBy(() -> appConfigUnderTest.getPublisherConfiguration(CHANGE_IDENTIFIER))
.hasMessageContaining(CHANGE_IDENTIFIER);
- Assertions.assertNull(appConfigUnderTest.getFtpesConfiguration());
+ Assertions.assertNull(appConfigUnderTest.getCertificateConfiguration());
}
@Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
index f49cd39..2013950 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
@@ -1,6 +1,6 @@
/*-
* ============LICENSE_START======================================================================
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -43,7 +43,7 @@
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
-public class DfcHttpClientTest {
+class DfcHttpClientTest {
private static final String USERNAME = "bob";
private static final String PASSWORD = "123";
@@ -55,28 +55,20 @@
DfcHttpClient dfcHttpClientSpy;
- private ImmutableFileServerData createFileServerData() {
- return ImmutableFileServerData.builder()
- .serverAddress(XNF_ADDRESS)
- .userId(USERNAME).password(PASSWORD)
- .port(PORT)
- .build();
- }
-
@BeforeEach
public void setup() {
dfcHttpClientSpy = spy(new DfcHttpClient(createFileServerData()));
}
@Test
- public void openConnection_successAuthSetup() throws DatafileTaskException {
+ void openConnection_successAuthSetup() throws DatafileTaskException {
dfcHttpClientSpy.open();
HttpClientConfig config = dfcHttpClientSpy.client.configuration();
assertEquals(HttpUtils.basicAuth(USERNAME, PASSWORD), config.headers().get("Authorization"));
}
@Test
- public void openConnection_failedBasicAuthSetupThrowException() {
+ void openConnection_failedBasicAuthSetupThrowException() {
ImmutableFileServerData serverData = ImmutableFileServerData.builder()
.serverAddress(XNF_ADDRESS)
.userId(USERNAME).password("")
@@ -90,7 +82,7 @@
}
@Test
- public void prepareUri_UriWithoutPort() {
+ void prepareUri_UriWithoutPort() {
ImmutableFileServerData serverData = ImmutableFileServerData.builder()
.serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD)
@@ -103,7 +95,7 @@
}
@Test
- public void collectFile_AllOk() throws Exception {
+ void collectFile_AllOk() throws Exception {
String REMOTE_FILE = "any";
Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes()));
@@ -121,7 +113,7 @@
}
@Test
- public void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException {
+ void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException {
String ERROR_RESPONSE = "This is unexpected message";
String REMOTE_FILE = "any";
Flux<Throwable> fis = Flux.error(new Throwable(ERROR_RESPONSE));
@@ -138,8 +130,16 @@
}
@Test
- public void isResponseOk_validateResponse() {
- assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_OK));
+ void isResponseOk_validateResponse() {
+ assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.NETTY_RESPONSE_OK));
assertFalse(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_ANY_NO_OK));
}
+
+ private ImmutableFileServerData createFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId(USERNAME).password(PASSWORD)
+ .port(PORT)
+ .build();
+ }
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
new file mode 100644
index 0000000..8df91d3
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
@@ -0,0 +1,164 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+@ExtendWith(MockitoExtension.class)
+class DfcHttpsClientTest {
+
+ private static final String USERNAME = "bob";
+ private static final String PASSWORD = "123";
+ private static final String XNF_ADDRESS = "127.0.0.1";
+ private static final int PORT = 443;
+ private static String remoteFile = "remoteFile";
+
+ @Mock
+ private PoolingHttpClientConnectionManager connectionManager;
+ @Mock
+ private Path localFile;
+
+ DfcHttpsClient dfcHttpsClientSpy;
+
+ @BeforeEach
+ public void setup() {
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(createFileServerData(), connectionManager));
+ }
+
+ @Test
+ void fileServerData_properLocationBasicAuth() throws Exception {
+ boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow();
+ assertEquals(true, result);
+ }
+
+ @Test
+ void fileServerData_properLocationNoBasicAuth() throws Exception {
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(emptyUserInFileServerData(), connectionManager));
+
+ boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow();
+ assertEquals(false, result);
+ }
+
+ @Test
+ void fileServerData_improperAuthDataExceptionOccurred() throws Exception {
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(invalidUserInFileServerData(), connectionManager));
+
+ assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow());
+ }
+
+ @Test
+ void dfcHttpsClient_flow_successfulCallAndResponseProcessing() throws Exception {
+ doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+ doReturn((long)3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class));
+
+ dfcHttpsClientSpy.open();
+ dfcHttpsClientSpy.collectFile(remoteFile, localFile);
+ dfcHttpsClientSpy.close();
+
+ verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class));
+ verify(dfcHttpsClientSpy, times(1))
+ .executeHttpClient(any(HttpGet.class));
+ verify(dfcHttpsClientSpy, times(1))
+ .processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile);
+ verify(dfcHttpsClientSpy, times(1))
+ .writeFile(eq(localFile), any(InputStream.class));
+ }
+
+ @Test
+ void dfcHttpsClient_flow_failedCallUnexpectedResponseCode() throws Exception {
+ doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+ doReturn(false).when(dfcHttpsClientSpy).isResponseOk(any(HttpResponse.class));
+
+ dfcHttpsClientSpy.open();
+
+ assertThrows(NonRetryableDatafileTaskException.class,
+ () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
+ }
+
+ @Test
+ void dfcHttpsClient_flow_failedCallConnectionTimeout() throws Exception {
+ doThrow(ConnectTimeoutException.class).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+
+ dfcHttpsClientSpy.open();
+
+ assertThrows(NonRetryableDatafileTaskException.class,
+ () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
+ }
+
+ @Test
+ void dfcHttpsClient_flow_failedCallIOExceptionForExecuteHttpClient() throws Exception {
+ doThrow(IOException.class).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+
+ dfcHttpsClientSpy.open();
+
+ assertThrows(DatafileTaskException.class,
+ () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
+ }
+
+ private ImmutableFileServerData createFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId(USERNAME).password(PASSWORD)
+ .port(PORT)
+ .build();
+ }
+
+ private ImmutableFileServerData emptyUserInFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("").password("")
+ .port(PORT)
+ .build();
+ }
+
+ private ImmutableFileServerData invalidUserInFileServerData() {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("demo").password("")
+ .port(PORT)
+ .build();
+ }
+}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
index 42ab4b3..0d52858 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java
@@ -20,16 +20,28 @@
import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.HttpVersion;
import io.netty.handler.codec.http.cookie.Cookie;
+import org.apache.http.Header;
+import org.apache.http.HeaderIterator;
+import org.apache.http.HttpEntity;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.params.HttpParams;
import reactor.netty.http.client.HttpClientResponse;
import reactor.util.context.Context;
import reactor.util.context.ContextView;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
public class HttpClientResponseHelper {
- public static final HttpClientResponse RESPONSE_OK = new HttpClientResponse() {
+ public static final HttpClientResponse NETTY_RESPONSE_OK = new HttpClientResponse() {
@Override
public Map<CharSequence, Set<Cookie>> cookies() {
@@ -167,4 +179,166 @@
return HttpResponseStatus.NOT_IMPLEMENTED;
}
};
+
+ public static final CloseableHttpResponse APACHE_RESPONSE_OK = new CloseableHttpResponse() {
+ @Override public void close() throws IOException {
+ getEntity().getContent().close();
+ }
+
+ @Override public StatusLine getStatusLine() {
+ return new StatusLine() {
+ @Override public ProtocolVersion getProtocolVersion() {
+ return null;
+ }
+
+ @Override public int getStatusCode() {
+ return 200;
+ }
+
+ @Override public String getReasonPhrase() {
+ return null;
+ }
+ };
+ }
+
+ @Override public void setStatusLine(StatusLine statusLine) {
+
+ }
+
+ @Override public void setStatusLine(ProtocolVersion protocolVersion, int i) {
+
+ }
+
+ @Override public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) {
+
+ }
+
+ @Override public void setStatusCode(int i) throws IllegalStateException {
+
+ }
+
+ @Override public void setReasonPhrase(String s) throws IllegalStateException {
+
+ }
+
+ @Override public HttpEntity getEntity() {
+ return new HttpEntity() {
+ @Override public boolean isRepeatable() {
+ return false;
+ }
+
+ @Override public boolean isChunked() {
+ return false;
+ }
+
+ @Override public long getContentLength() {
+ return 0;
+ }
+
+ @Override public Header getContentType() {
+ return null;
+ }
+
+ @Override public Header getContentEncoding() {
+ return null;
+ }
+
+ @Override public InputStream getContent() throws IOException, UnsupportedOperationException {
+ return new ByteArrayInputStream("abc".getBytes());
+ }
+
+ @Override public void writeTo(OutputStream outputStream) throws IOException {
+
+ }
+
+ @Override public boolean isStreaming() {
+ return false;
+ }
+
+ @Override public void consumeContent() throws IOException {
+
+ }
+ };
+ }
+
+ @Override public void setEntity(HttpEntity httpEntity) {
+
+ }
+
+ @Override public Locale getLocale() {
+ return null;
+ }
+
+ @Override public void setLocale(Locale locale) {
+
+ }
+
+ @Override public ProtocolVersion getProtocolVersion() {
+ return null;
+ }
+
+ @Override public boolean containsHeader(String s) {
+ return false;
+ }
+
+ @Override public Header[] getHeaders(String s) {
+ return new Header[0];
+ }
+
+ @Override public Header getFirstHeader(String s) {
+ return null;
+ }
+
+ @Override public Header getLastHeader(String s) {
+ return null;
+ }
+
+ @Override public Header[] getAllHeaders() {
+ return new Header[0];
+ }
+
+ @Override public void addHeader(Header header) {
+
+ }
+
+ @Override public void addHeader(String s, String s1) {
+
+ }
+
+ @Override public void setHeader(Header header) {
+
+ }
+
+ @Override public void setHeader(String s, String s1) {
+
+ }
+
+ @Override public void setHeaders(Header[] headers) {
+
+ }
+
+ @Override public void removeHeader(Header header) {
+
+ }
+
+ @Override public void removeHeaders(String s) {
+
+ }
+
+ @Override public HeaderIterator headerIterator() {
+ return null;
+ }
+
+ @Override public HeaderIterator headerIterator(String s) {
+ return null;
+ }
+
+ @Override public HttpParams getParams() {
+ return null;
+ }
+
+ @Override public void setParams(HttpParams httpParams) {
+
+ }
+ };
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java
new file mode 100644
index 0000000..fa4473a
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java
@@ -0,0 +1,52 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.http;
+
+import org.junit.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@ExtendWith(MockitoExtension.class)
+public class HttpsClientConnectionManagerUtilTest {
+
+ private static final String KEY_PATH = "src/test/resources/keystore.p12";
+ private static final String KEY_PASSWORD = "src/test/resources/keystore.pass";
+ private static final String KEY_IMPROPER_PASSWORD = "src/test/resources/dfc.jks.pass";
+ private static final String TRUSTED_CA_PATH = "src/test/resources/trust.jks";
+ private static final String TRUSTED_CA_PASSWORD = "src/test/resources/trust.pass";
+
+ @Test
+ public void emptyManager_shouldThrowException() {
+ assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance());
+ }
+
+ @Test
+ public void creatingManager_successfulCase() throws Exception {
+ HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD);
+ assertNotNull(HttpsClientConnectionManagerUtil.instance());
+ }
+
+ @Test
+ public void creatingManager_improperSecretShouldThrowException() {
+ assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_IMPROPER_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD));
+ assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance());
+ }
+
+}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
index fb47579..61adef9 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java
@@ -34,6 +34,7 @@
assertEquals(Scheme.FTPES, Scheme.getSchemeFromString("FTPES"));
assertEquals(Scheme.SFTP, Scheme.getSchemeFromString("SFTP"));
assertEquals(Scheme.HTTP, Scheme.getSchemeFromString("HTTP"));
+ assertEquals(Scheme.HTTPS, Scheme.getSchemeFromString("HTTPS"));
}
@Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
index 1146cf2..ceb8a98 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java
@@ -38,12 +38,14 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
-import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
+import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient;
+import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient;
import org.onap.dcaegen2.collectors.datafile.model.Counters;
import org.onap.dcaegen2.collectors.datafile.model.FileData;
import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
@@ -64,6 +66,8 @@
private static final String FILE_READY_CHANGE_TYPE = "FileReady";
private static final String FTPES_SCHEME = "ftpes://";
private static final String SFTP_SCHEME = "sftp://";
+ private static final String HTTP_SCHEME = "http://";
+ private static final String HTTPS_SCHEME = "https://";
private static final String SERVER_ADDRESS = "192.168.0.101";
private static final int PORT_22 = 22;
private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz";
@@ -79,24 +83,36 @@
private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
+ private static final String HTTP_LOCATION =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
+ private static final String HTTP_LOCATION_NO_PORT =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
+ private static final String HTTPS_LOCATION =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
+ private static final String HTTPS_LOCATION_NO_PORT =
+ HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
+
private static final String GZIP_COMPRESSION = "gzip";
private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
private static final String FILE_FORMAT_VERSION = "V10";
- private static final String FTP_KEY_PATH = "ftpKeyPath";
- private static final String FTP_KEY_PASSWORD_PATH = "ftpKeyPassword";
+ private static final String CERTIFICATE_KEY_PATH = "certificateKeyPath";
+ private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword";
private static final String TRUSTED_CA_PATH = "trustedCAPath";
private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword";
private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES";
private static AppConfig appConfigMock = mock(AppConfig.class);
- private static FtpesConfig ftpesConfigMock = mock(FtpesConfig.class);
+ private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class);
private FtpesClient ftpesClientMock = mock(FtpesClient.class);
private SftpClient sftpClientMock = mock(SftpClient.class);
private final Map<String, String> contextMap = new HashMap<>();
+ private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class);
+ private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class);
+
private Counters counters;
private MessageMetaData createMessageMetaData() {
@@ -145,11 +161,11 @@
@BeforeAll
static void setUpConfiguration() {
- when(appConfigMock.getFtpesConfiguration()).thenReturn(ftpesConfigMock);
- when(ftpesConfigMock.keyCert()).thenReturn(FTP_KEY_PATH);
- when(ftpesConfigMock.keyPasswordPath()).thenReturn(FTP_KEY_PASSWORD_PATH);
- when(ftpesConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH);
- when(ftpesConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH);
+ when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock);
+ when(certificateConfigMock.keyCert()).thenReturn(CERTIFICATE_KEY_PATH);
+ when(certificateConfigMock.keyPasswordPath()).thenReturn(CERTIFICATE_KEY_PASSWORD_PATH);
+ when(certificateConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH);
+ when(certificateConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH);
}
@BeforeEach
@@ -178,6 +194,7 @@
assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
@Test
@@ -210,6 +227,70 @@
}
@Test
+ public void whenHttpFile_returnCorrectResponse() throws Exception {
+ FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
+ doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any());
+
+ FileData fileData = createFileData(HTTP_LOCATION_NO_PORT, Scheme.HTTP);
+
+ FilePublishInformation expectedfilePublishInformation =
+ createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ // The same again, but with port
+ fileData = createFileData(HTTP_LOCATION, Scheme.HTTP);
+ expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ verify(dfcHttpClientMock, times(2)).open();
+ verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(dfcHttpClientMock, times(2)).close();
+ verifyNoMoreInteractions(dfcHttpClientMock);
+
+ assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
+ assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
+ }
+
+ @Test
+ public void whenHttpsFile_returnCorrectResponse() throws Exception {
+ FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
+ doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any());
+
+ FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT, Scheme.HTTPS);
+
+ FilePublishInformation expectedfilePublishInformation =
+ createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ // The same again, but with port
+ fileData = createFileData(HTTPS_LOCATION, Scheme.HTTPS);
+ expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION);
+
+ StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
+ .expectNext(expectedfilePublishInformation) //
+ .verifyComplete();
+
+ verify(dfcHttpsClientMock, times(2)).open();
+ verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(dfcHttpsClientMock, times(2)).close();
+ verifyNoMoreInteractions(dfcHttpsClientMock);
+
+ assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
+ assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
+ }
+
+ @Test
public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception {
FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
@@ -226,6 +307,7 @@
assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
assertEquals(4, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 4");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
@Test
@@ -245,6 +327,7 @@
assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
@Test
@@ -267,5 +350,6 @@
assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
+ assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
}
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
index 09d7627..85f52ff 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/ScheduledTasksTest.java
@@ -392,6 +392,7 @@
assertEquals(2, testedObject.getCounters().getTotalReceivedEvents(),"totalReceivedEvents should have been 2");
assertEquals(1, testedObject.getCounters().getNoOfFailedFtp(),"failedFtp should have been 1");
+ assertEquals(0, testedObject.getCounters().getNoOfFailedHttp(),"failedHttp should have been 0");
}
@Test
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java
new file mode 100644
index 0000000..75ad8a4
--- /dev/null
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/SecurityUtilTest.java
@@ -0,0 +1,120 @@
+/*-
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2021 Nokia. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+package org.onap.dcaegen2.collectors.datafile.utils;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+/**
+ * Tests the SecurityUtil.
+ *
+ * @author <a href="mailto:krzysztof.gajewski@nokia.com">Krzysztof Gajewski</a>
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(SecurityUtil.class)
+@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"})
+public class SecurityUtilTest {
+
+ @Mock
+ Path path;
+
+ private static final String expectedPassword = "password";
+ private static final String validPath = "/validPath";
+ private static final String invalidPath = "/invalidPath";
+
+ @Test
+ public void whenGetKeystorePasswordFromFile_passwordSuccessfullyReturned() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenReturn("password".getBytes());
+
+ String result = SecurityUtil.getKeystorePasswordFromFile(validPath);
+ assertEquals(expectedPassword, result);
+
+ }
+
+ @Test
+ public void whenGetKeystorePasswordFromFile_IOExceptionForValidPath() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenThrow(IOException.class);
+
+ assertEquals("", SecurityUtil.getKeystorePasswordFromFile(validPath));
+
+ }
+
+ @Test
+ public void whenGetKeystorePasswordFromFile_InvalidPathExceptionForInvalidPath() {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(invalidPath)).thenThrow(InvalidPathException.class);
+
+ assertThrows(InvalidPathException.class, () -> SecurityUtil.getKeystorePasswordFromFile(invalidPath));
+
+ }
+
+ @Test
+ public void whenGetTruststorePasswordFromFile_passwordSuccessfullyReturned() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenReturn("password".getBytes());
+
+ String result = SecurityUtil.getTruststorePasswordFromFile(validPath);
+ assertEquals(expectedPassword, result);
+
+ }
+
+ @Test
+ public void whenGetTruststorePasswordFromFile_IOExceptionForValidPath() throws Exception {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(validPath)).thenReturn(path);
+ when(Files.readAllBytes(path)).thenThrow(IOException.class);
+
+ assertEquals("", SecurityUtil.getTruststorePasswordFromFile(validPath));
+
+ }
+
+ @Test
+ public void whenGetTruststorePasswordFromFile_InvalidPathExceptionForInvalidPath() {
+ mockStatic(Paths.class);
+ mockStatic(Files.class);
+ when(Paths.get(invalidPath)).thenThrow(InvalidPathException.class);
+
+ assertThrows(InvalidPathException.class, () -> SecurityUtil.getTruststorePasswordFromFile(invalidPath));
+
+ }
+
+}
diff --git a/datafile-app-server/src/test/resources/datafile_endpoints_test.json b/datafile-app-server/src/test/resources/datafile_endpoints_test.json
index 8e51b80..00b2488 100644
--- a/datafile-app-server/src/test/resources/datafile_endpoints_test.json
+++ b/datafile-app-server/src/test/resources/datafile_endpoints_test.json
@@ -1,10 +1,10 @@
{
"config": {
"//description": "This file is only used for testing purposes",
- "dmaap.ftpesConfig.keyCert": "/src/test/resources/dfc.jks",
- "dmaap.ftpesConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
- "dmaap.ftpesConfig.trustedCa": "/src/test/resources/ftp.jks",
- "dmaap.ftpesConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass",
+ "dmaap.certificateConfig.keyCert": "/src/test/resources/dfc.jks",
+ "dmaap.certificateConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
+ "dmaap.certificateConfig.trustedCa": "/src/test/resources/cert.jks",
+ "dmaap.certificateConfig.trustedCaPasswordPath": "/src/test/resources/cert.jks.pass",
"dmaap.security.trustStorePath": "src/test/resources/trust.jks",
"dmaap.security.trustStorePasswordPath": "src/test/resources/trust.pass",
"dmaap.security.keyStorePath": "src/test/resources/cert.jks",
diff --git a/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json b/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json
index e132746..73bb744 100644
--- a/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json
+++ b/datafile-app-server/src/test/resources/datafile_endpoints_test_2producers.json
@@ -1,10 +1,10 @@
{
"config": {
"//description": "This file is only used for testing purposes",
- "dmaap.ftpesConfig.keyCert": "/src/test/resources/dfc.jks",
- "dmaap.ftpesConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
- "dmaap.ftpesConfig.trustedCa": "/src/test/resources/ftp.jks",
- "dmaap.ftpesConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass",
+ "dmaap.certificateConfig.keyCert": "/src/test/resources/dfc.jks",
+ "dmaap.certificateConfig.keyPasswordPath": "/src/test/resources/dfc.jks.pass",
+ "dmaap.certificateConfig.trustedCa": "/src/test/resources/ftp.jks",
+ "dmaap.certificateConfig.trustedCaPasswordPath": "/src/test/resources/ftp.jks.pass",
"dmaap.security.trustStorePath": "src/test/resources/trust.jks",
"dmaap.security.trustStorePasswordPath": "src/test/resources/trust.pass",
"dmaap.security.keyStorePath": "src/test/resources/cert.jks",
diff --git a/datafile-app-server/src/test/resources/keystore.p12 b/datafile-app-server/src/test/resources/keystore.p12
new file mode 100644
index 0000000..b847707
--- /dev/null
+++ b/datafile-app-server/src/test/resources/keystore.p12
Binary files differ
diff --git a/datafile-app-server/src/test/resources/keystore.pass b/datafile-app-server/src/test/resources/keystore.pass
new file mode 100644
index 0000000..1e7befc
--- /dev/null
+++ b/datafile-app-server/src/test/resources/keystore.pass
@@ -0,0 +1 @@
+HVpAf0kHGl4P#fdpblJLka6b
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 7f328cd..3a2fade 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ ============LICENSE_START=====================================================================
- ~ Copyright (C) 2018-2020 NOKIA Intellectual Property. All rights reserved.
+ ~ Copyright (C) 2018-2021 NOKIA Intellectual Property. All rights reserved.
~ Copyright (C) 2018-2021 Nordix Foundation. All rights reserved.
~ ==============================================================================================
~ Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +31,7 @@
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.3-SNAPSHOT</version>
<name>dcaegen2-collectors.datafile</name>
<description>datafile collector</description>
@@ -74,6 +74,7 @@
<springfox.version>3.0.0</springfox.version>
<awaitility.version>3.1.6</awaitility.version>
<jackson-databind.version>2.11.4</jackson-databind.version>
+ <powermock.version>2.0.9</powermock.version>
<!-- Plugin versions -->
<maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
@@ -235,6 +236,18 @@
<version>${awaitility.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
<!--REQUIRED TO GENERATE DOCUMENTATION -->
<dependency>
diff --git a/version.properties b/version.properties
index 80408f3..8e26230 100644
--- a/version.properties
+++ b/version.properties
@@ -1,6 +1,6 @@
major=1
minor=5
-patch=2
+patch=3
base_version=${major}.${minor}.${patch}
release_version=${base_version}
snapshot_version=${base_version}-SNAPSHOT