Add JWT support in HTTP/HTTPS based locations
Issue-ID: DCAEGEN2-2536
Signed-off-by: Krzysztof Gajewski <krzysztof.gajewski@nokia.com>
Change-Id: I47a928159853333014b0fd413a085b7c50eeb7a0
diff --git a/Changelog.md b/Changelog.md
index d678e9b..72a8b53 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,12 @@
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
+## [1.5.4] - 23/02/2021
+### Added
+- JWT token support for HTTP/HTTPS
+### Changed
+- FileData / FileServerData prepared to store uri elements
+
## [1.5.3] - 11/02/2021
### Added
- HTTPS support for DFC
diff --git a/datafile-app-server/pom.xml b/datafile-app-server/pom.xml
index 5e0330b..64b6f65 100644
--- a/datafile-app-server/pom.xml
+++ b/datafile-app-server/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.5.3-SNAPSHOT</version>
+ <version>1.5.4-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
@@ -116,6 +116,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5</artifactId>
+ </dependency>
<!-- Actuator dependencies -->
<dependency>
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java
index 32241fd..2c44abb 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START======================================================================
* Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
+ * Modifications copyright (C) 2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -16,12 +17,15 @@
package org.onap.dcaegen2.collectors.datafile.commons;
+import java.util.List;
import java.util.Optional;
+import org.apache.hc.core5.http.NameValuePair;
import org.immutables.value.Value;
/**
* Data about the file server to collect a file from.
+ * In case of http protocol it also contains data required to recreate target uri
*
* @author <a href="mailto:henrik.b.andersson@est.tech">Henrik Andersson</a>
*
@@ -37,4 +41,9 @@
public String password();
public Optional<Integer> port();
+
+ @Value.Redacted
+ public Optional<List<NameValuePair>> queryParameters();
+
+ public Optional<String> uriRawFragment();
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
index 3ccc9fb..4564a44 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java
@@ -19,6 +19,7 @@
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.service.HttpUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,17 +62,22 @@
@Override public void open() throws DatafileTaskException {
logger.trace("Setting httpClient for file download.");
- basicAuthDataPresentOrThrow();
+ String authorizationContent = getAuthorizationContent();
this.client = HttpClient.create(pool).keepAlive(true).headers(
- h -> h.add("Authorization", HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password())));
+ h -> h.add("Authorization", authorizationContent));
logger.trace("httpClient, auth header was set.");
}
- private void basicAuthDataPresentOrThrow() throws DatafileTaskException {
- if ((this.fileServerData.userId().isEmpty()) || (this.fileServerData.password().isEmpty())) {
+ protected String getAuthorizationContent() throws DatafileTaskException {
+ String jwtToken = HttpUtils.getJWTToken(fileServerData);
+ if (!jwtToken.isEmpty()) {
+ return HttpUtils.jwtAuthContent(jwtToken);
+ }
+ if (!HttpUtils.isBasicAuthDataFilled(fileServerData)) {
throw new DatafileTaskException("Not sufficient basic auth data for file.");
}
+ return HttpUtils.basicAuthContent(this.fileServerData.userId(), this.fileServerData.password());
}
@Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException {
@@ -92,7 +98,10 @@
}
if (isDownloadFailed(errorMessage)) {
- throw new DatafileTaskException("Error occured during datafile download: ", errorMessage.get());
+ if (errorMessage.get() instanceof NonRetryableDatafileTaskException) {
+ throw (NonRetryableDatafileTaskException) errorMessage.get();
+ }
+ throw (DatafileTaskException) errorMessage.get();
}
logger.trace("HTTP collectFile OK");
@@ -104,7 +113,11 @@
@NotNull protected Consumer<Throwable> processFailedConnectionWithServer(CountDownLatch latch, AtomicReference<Exception> errorMessages) {
return (Throwable response) -> {
- errorMessages.set(new Exception("Error in connection has occurred during file download", response));
+ Exception e = new Exception("Error in connection has occurred during file download", response);
+ errorMessages.set(new DatafileTaskException(response.getMessage(), e));
+ if (response instanceof NonRetryableDatafileTaskException) {
+ errorMessages.set(new NonRetryableDatafileTaskException(response.getMessage(), e));
+ }
latch.countDown();
};
}
@@ -119,7 +132,7 @@
logger.trace("CollectFile fetched: {}", localFile);
response.close();
} catch (IOException e) {
- errorMessages.set(new Exception("Error fetching file with", e));
+ errorMessages.set(new DatafileTaskException("Error fetching file with", e));
} finally {
latch.countDown();
}
@@ -128,24 +141,31 @@
protected Flux<InputStream> getServerResponse(String remoteFile) {
return client.get()
- .uri(prepareUri(remoteFile))
+ .uri(HttpUtils.prepareHttpUri(fileServerData, remoteFile))
.response((responseReceiver, byteBufFlux) -> {
logger.trace("HTTP response status - {}", responseReceiver.status());
if(isResponseOk(responseReceiver)){
return byteBufFlux.aggregate().asInputStream();
}
- return Mono.error(new Throwable("Unexpected server response code - "
- + responseReceiver.status().toString()));
+ if (isErrorInConnection(responseReceiver)) {
+ return Mono.error(new NonRetryableDatafileTaskException(
+ HttpUtils.nonRetryableResponse(getResponseCode(responseReceiver))));
+ }
+ return Mono.error(new DatafileTaskException(
+ HttpUtils.retryableResponse(getResponseCode(responseReceiver))));
});
}
protected boolean isResponseOk(HttpClientResponse httpClientResponse) {
- return httpClientResponse.status().code() == 200;
+ return getResponseCode(httpClientResponse) == 200;
}
- @NotNull protected String prepareUri(String remoteFile) {
- int port = fileServerData.port().orElse(HttpUtils.HTTP_DEFAULT_PORT);
- return "http://" + fileServerData.serverAddress() + ":" + port + remoteFile;
+ private int getResponseCode(HttpClientResponse responseReceiver) {
+ return responseReceiver.status().code();
+ }
+
+ protected boolean isErrorInConnection(HttpClientResponse httpClientResponse) {
+ return getResponseCode(httpClientResponse) >= 400;
}
@Override public void close() {
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
index 3090815..c2d72f6 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java
@@ -27,7 +27,6 @@
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
-import org.jetbrains.annotations.NotNull;
import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient;
import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
@@ -85,10 +84,11 @@
@Override public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException {
logger.trace("Prepare to collectFile {}", localFile);
- HttpGet httpGet = new HttpGet(prepareUri(remoteFile));
- if (basicAuthValidNotPresentOrThrow()) {
- httpGet.addHeader("Authorization",
- HttpUtils.basicAuth(this.fileServerData.userId(), this.fileServerData.password()));
+ HttpGet httpGet = new HttpGet(HttpUtils.prepareHttpsUri(fileServerData, remoteFile));
+
+ String authorizationContent = getAuthorizationContent();
+ if (!authorizationContent.isEmpty()) {
+ httpGet.addHeader("Authorization", authorizationContent);
}
try {
HttpResponse httpResponse = makeCall(httpGet);
@@ -99,11 +99,23 @@
logger.trace("HTTPS collectFile OK");
}
+ private String getAuthorizationContent() throws DatafileTaskException {
+ String jwtToken = HttpUtils.getJWTToken(fileServerData);
+ if (shouldUseBasicAuth(jwtToken)) {
+ return HttpUtils.basicAuthContent(this.fileServerData.userId(), this.fileServerData.password());
+ }
+ return HttpUtils.jwtAuthContent(jwtToken);
+ }
+
+ private boolean shouldUseBasicAuth(String jwtToken) throws DatafileTaskException {
+ return basicAuthValidNotPresentOrThrow() && jwtToken.isEmpty();
+ }
+
protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException {
if (isAuthDataEmpty()) {
return false;
}
- if (isAuthDataFilled()) {
+ if (HttpUtils.isBasicAuthDataFilled(fileServerData)) {
return true;
}
throw new DatafileTaskException("Not sufficient basic auth data for file.");
@@ -113,15 +125,6 @@
return this.fileServerData.userId().isEmpty() && this.fileServerData.password().isEmpty();
}
- private boolean isAuthDataFilled() {
- return !this.fileServerData.userId().isEmpty() && !this.fileServerData.password().isEmpty();
- }
-
- @NotNull protected String prepareUri(String remoteFile) {
- int port = fileServerData.port().orElse(HttpUtils.HTTPS_DEFAULT_PORT);
- return "https://" + fileServerData.serverAddress() + ":" + port + remoteFile;
- }
-
protected HttpResponse makeCall(HttpGet httpGet)
throws IOException, DatafileTaskException {
try {
@@ -131,10 +134,10 @@
}
EntityUtils.consume(httpResponse.getEntity());
- throw new NonRetryableDatafileTaskException(
- "Unexpected response code - " + httpResponse.getStatusLine().getStatusCode()
- + ". No retry attempts will be done.");
-
+ if (isErrorInConnection(httpResponse)) {
+ throw new NonRetryableDatafileTaskException(HttpUtils.retryableResponse(getResponseCode(httpResponse)));
+ }
+ throw new DatafileTaskException(HttpUtils.nonRetryableResponse(getResponseCode(httpResponse)));
} catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException e) {
throw new NonRetryableDatafileTaskException(
"Unable to get file from xNF. No retry attempts will be done.", e);
@@ -146,8 +149,16 @@
return httpsClient.execute(httpGet);
}
- protected boolean isResponseOk(HttpResponse response) {
- return response.getStatusLine().getStatusCode() == 200;
+ protected boolean isResponseOk(HttpResponse httpResponse) {
+ return getResponseCode(httpResponse) == 200;
+ }
+
+ private int getResponseCode(HttpResponse httpResponse) {
+ return httpResponse.getStatusLine().getStatusCode();
+ }
+
+ protected boolean isErrorInConnection(HttpResponse httpResponse) {
+ return getResponseCode(httpResponse) >= 400;
}
protected void processResponse(HttpResponse response, Path localFile) throws IOException {
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java
index 1c8f57d..e7d2e15 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java
@@ -1,6 +1,7 @@
/*-
* ============LICENSE_START=======================================================
* Copyright (C) 2019 Nordix Foundation.
+ * Copyright (C) 2021 Nokia. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,8 +24,11 @@
import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.List;
import java.util.Optional;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
import org.immutables.gson.Gson;
import org.immutables.value.Value;
import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
@@ -101,6 +105,7 @@
/**
* Get the data about the file server where the file should be collected from.
+ * Query data included as it can contain JWT token
*
* @return the data about the file server where the file should be collected from.
*/
@@ -114,6 +119,15 @@
if (uri.getPort() > 0) {
builder.port(uri.getPort());
}
+ URIBuilder uriBuilder = new URIBuilder(uri);
+ List<NameValuePair> query = uriBuilder.getQueryParams();
+ if (query != null && !query.isEmpty()) {
+ builder.queryParameters(query);
+ }
+ String fragment = uri.getRawFragment();
+ if (fragment != null && fragment.length() > 0) {
+ builder.uriRawFragment(fragment);
+ }
return builder.build();
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
index e2c1e2f..69bfcf4 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START======================================================================
* Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
- * Modifications Copyright (C) 2020 Nokia. All rights reserved
+ * Modifications Copyright (C) 2020-2021 Nokia. All rights reserved
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,23 +19,208 @@
package org.onap.dcaegen2.collectors.datafile.service;
+import org.apache.hc.core5.http.NameValuePair;
import org.apache.http.HttpStatus;
+import org.jetbrains.annotations.NotNull;
+import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
+import java.util.ArrayList;
import java.util.Base64;
+import java.util.List;
+import java.util.Optional;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public final class HttpUtils implements HttpStatus {
+ private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
public static final int HTTP_DEFAULT_PORT = 80;
public static final int HTTPS_DEFAULT_PORT = 443;
+ public static final String JWT_TOKEN_NAME = "access_token";
+ public static final String AUTH_JWT_WARN = "Both JWT token and Basic auth data present. Omitting basic auth info.";
+ public static final String AUTH_JWT_ERROR = "More than one JWT token present in the queryParameters. Omitting JWT token.";
private HttpUtils() {
}
- public static boolean isSuccessfulResponseCode(Integer statusCode) {
+ @NotNull
+ public static String nonRetryableResponse(int responseCode) {
+ return "Unexpected response code - " + responseCode;
+ }
+
+ @NotNull
+ public static String retryableResponse(int responseCode) {
+ return "Unexpected response code - " + responseCode + ". No retry attempts will be done.";
+ }
+
+ public static boolean isSuccessfulResponseCodeWithDataRouter(Integer statusCode) {
return statusCode >= 200 && statusCode < 300;
}
- public static String basicAuth(String username, String password) {
+ public static boolean isBasicAuthDataFilled(final FileServerData fileServerData) {
+ return !fileServerData.userId().isEmpty() && !fileServerData.password().isEmpty();
+ }
+
+ public static String basicAuthContent(String username, String password) {
return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
}
+
+ public static String jwtAuthContent(String token) {
+ return "Bearer " + token;
+ }
+
+ /**
+ * Prepare uri to retrieve file from xNF using HTTP connection. If JWT token was included
+ * in the queryParameters, it is removed. Other entries are rewritten.
+ *
+ * @param fileServerData fileServerData including - server address, port, queryParameters and uriRawFragment
+ * @param remoteFile file which has to be downloaded
+ * @return uri String representing the xNF HTTP location
+ */
+ @NotNull public static String prepareHttpUri(FileServerData fileServerData, String remoteFile){
+ return prepareUri("http", fileServerData, remoteFile, HTTP_DEFAULT_PORT);
+ }
+
+ /**
+ * Prepare uri to retrieve file from xNF using HTTPS connection. If JWT token was included
+ * in the queryParameters, it is removed. Other entries are rewritten.
+ *
+ * @param fileServerData fileServerData including - server address, port, queryParameters and uriRawFragment
+ * @param remoteFile file which has to be downloaded
+ * @return uri String representing the xNF HTTPS location
+ */
+ @NotNull public static String prepareHttpsUri(FileServerData fileServerData, String remoteFile){
+ return prepareUri("https", fileServerData, remoteFile, HTTPS_DEFAULT_PORT);
+ }
+
+ /**
+ * Prepare uri to retrieve file from xNF. If JWT token was included
+ * in the queryParameters, it is removed. Other entries are rewritten.
+ *
+ * @param scheme scheme which is used during the connection
+ * @param fileServerData fileServerData including - server address, port, query and fragment
+ * @param remoteFile file which has to be downloaded
+ * @param defaultPort default port which exchange empty entry for given connection type
+ * @return uri String representing the xNF location
+ */
+ @NotNull public static String prepareUri(String scheme, FileServerData fileServerData, String remoteFile, int defaultPort) {
+ int port = fileServerData.port().orElse(defaultPort);
+ String query = rewriteQueryWithoutToken(fileServerData.queryParameters().orElse(new ArrayList<>()));
+ String fragment = fileServerData.uriRawFragment().orElse("");
+ if (!query.isEmpty()) {
+ query = "?" + query;
+ }
+ if (!fragment.isEmpty()) {
+ fragment = "#" + fragment;
+ }
+ return scheme + "://" + fileServerData.serverAddress() + ":" + port + remoteFile + query + fragment;
+ }
+
+ /**
+ * Returns JWT token string (if single exist) from the queryParameters.
+ *
+ * @param fileServerData file server data which contain queryParameters where JWT token may exist
+ * @return JWT token value if single token entry exist or empty string elsewhere.
+ * If JWT token key has no value, empty string will be returned.
+ */
+ public static String getJWTToken(FileServerData fileServerData) {
+ Optional<List<NameValuePair>> query = fileServerData.queryParameters();
+ if (!query.isPresent()) {
+ return "";
+ }
+ List<NameValuePair> queryParameters = query.get();
+ if (queryParameters.isEmpty()) {
+ return "";
+ }
+ boolean jwtTokenKeyPresent = HttpUtils.isQueryWithSingleJWT(queryParameters);
+ if (!jwtTokenKeyPresent) {
+ return "";
+ }
+ String token = HttpUtils.getJWTToken(query.get());
+ if (HttpUtils.isBasicAuthDataFilled(fileServerData)) {
+ logger.warn(HttpUtils.AUTH_JWT_WARN);
+ }
+ return token;
+ }
+
+ /**
+ * Checks if the queryParameters contains single JWT token entry. Valid queryParameters
+ * contains only one token entry.
+ *
+ * @param query queryParameters
+ * @return true if queryParameters contains single token
+ */
+ public static boolean isQueryWithSingleJWT(List<NameValuePair> query) {
+ if (query == null) {
+ return false;
+ }
+ int i = getJWTTokenCount(query);
+ if (i == 0) {
+ return false;
+ }
+ if (i > 1) {
+ logger.error(AUTH_JWT_ERROR);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns the number of JWT token entries. Valid queryParameters contains only one token entry.
+ *
+ * @param queryElements elements of the queryParameters
+ * @return true if queryParameters contains single JWT token entry
+ */
+ public static int getJWTTokenCount(List<NameValuePair> queryElements) {
+ int i = 0;
+ for (NameValuePair element : queryElements) {
+ if (element.getName().equals(JWT_TOKEN_NAME)) {
+ i++;
+ }
+ }
+ return i;
+ }
+
+ private static String getJWTToken(List<NameValuePair> query) {
+ for (NameValuePair element : query) {
+ if (!element.getName().equals(JWT_TOKEN_NAME)) {
+ continue;
+ }
+ if (element.getValue() != null) {
+ return element.getValue();
+ }
+ return "";
+ }
+ return "";
+ }
+
+ /**
+ * Rewrites HTTP queryParameters without JWT token
+ *
+ * @param query list of NameValuePair of elements sent in the queryParameters
+ * @return String representation of queryParameters elements which were provided in the input
+ * Empty string is possible when queryParameters is empty or contains only access_token key.
+ */
+ public static String rewriteQueryWithoutToken(List<NameValuePair> query) {
+ if (query.isEmpty()) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (NameValuePair nvp : query) {
+ if (nvp.getName().equals(JWT_TOKEN_NAME)) {
+ continue;
+ }
+ sb.append(nvp.getName());
+ if (nvp.getValue() != null) {
+ sb.append("=");
+ sb.append(nvp.getValue());
+ }
+ sb.append("&");
+ }
+ if ((sb.length() > 0) && (sb.charAt(sb.length() - 1 ) == '&')) {
+ sb.deleteCharAt(sb.length() - 1 );
+ }
+ return sb.toString();
+ }
}
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java
index c24c6c9..ef2341f 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/DataRouterPublisher.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START=======================================================
* Copyright (C) 2019 Nordix Foundation.
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -133,7 +133,7 @@
private Mono<FilePublishInformation> handleHttpResponse(HttpStatus response, FilePublishInformation publishInfo) {
MDC.setContextMap(publishInfo.getContext());
- if (HttpUtils.isSuccessfulResponseCode(response.value())) {
+ if (HttpUtils.isSuccessfulResponseCodeWithDataRouter(response.value())) {
counters.incTotalPublishedFiles();
logger.trace("Publishing file {} to DR successful!", publishInfo.getName());
return Mono.just(publishInfo);
diff --git a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
index cfc7754..7038043 100644
--- a/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
+++ b/datafile-app-server/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java
@@ -1,7 +1,7 @@
/*-
* ============LICENSE_START======================================================================
* Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
- * Copyright (C) 2020 Nokia. All rights reserved.
+ * Copyright (C) 2020-2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -111,7 +111,7 @@
counters.incNoOfCollectedFiles();
return Mono.just(Optional.of(getFilePublishInformation(fileData, localFile, context)));
} catch (NonRetryableDatafileTaskException nre) {
- logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.name(), nre);
+ logger.warn("Failed to download file: {} {}, reason: ", fileData.sourceName(), fileData.name(), nre);
incFailedAttemptsCounter(fileData);
return Mono.just(Optional.empty()); // Give up
} catch (DatafileTaskException e) {
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
index 2013950..98804a0 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java
@@ -15,6 +15,7 @@
*/
package org.onap.dcaegen2.collectors.datafile.http;
+import org.apache.hc.core5.net.URIBuilder;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -29,6 +30,7 @@
import java.io.ByteArrayInputStream;
import java.io.InputStream;
+import java.net.URISyntaxException;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -49,6 +51,8 @@
private static final String PASSWORD = "123";
private static final String XNF_ADDRESS = "127.0.0.1";
private static final int PORT = 80;
+ private static final String JWT_PASSWORD = "thisIsThePassword";
+ private static String ACCESS_TOKEN = "access_token";
@Mock
private Path pathMock;
@@ -61,10 +65,10 @@
}
@Test
- void openConnection_successAuthSetup() throws DatafileTaskException {
+ void openConnection_successBasicAuthSetup() throws DatafileTaskException {
dfcHttpClientSpy.open();
HttpClientConfig config = dfcHttpClientSpy.client.configuration();
- assertEquals(HttpUtils.basicAuth(USERNAME, PASSWORD), config.headers().get("Authorization"));
+ assertEquals(HttpUtils.basicAuthContent(USERNAME, PASSWORD), config.headers().get("Authorization"));
}
@Test
@@ -81,18 +85,6 @@
.hasMessageContaining("Not sufficient basic auth data for file.");
}
- @Test
- void prepareUri_UriWithoutPort() {
- ImmutableFileServerData serverData = ImmutableFileServerData.builder()
- .serverAddress(XNF_ADDRESS)
- .userId(USERNAME).password(PASSWORD)
- .build();
- DfcHttpClient clientNoPortSpy = spy(new DfcHttpClient(serverData));
- String REMOTE_FILE = "any";
-
- String retrievedUri = clientNoPortSpy.prepareUri(REMOTE_FILE);
- assertTrue(retrievedUri.startsWith("http://" + XNF_ADDRESS + ":80"));
- }
@Test
void collectFile_AllOk() throws Exception {
@@ -113,6 +105,27 @@
}
@Test
+ void collectFile_AllOkWithJWTToken() throws Exception {
+ dfcHttpClientSpy = spy(new DfcHttpClient(fileServerDataWithJWTToken()));
+ String REMOTE_FILE = "any";
+ Flux<InputStream> fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes()));
+
+ dfcHttpClientSpy.open();
+ HttpClientConfig config = dfcHttpClientSpy.client.configuration();
+ assertEquals(HttpUtils.jwtAuthContent(JWT_PASSWORD), config.headers().get("Authorization"));
+
+ when(dfcHttpClientSpy.getServerResponse(any())).thenReturn(fis);
+ doReturn(false).when(dfcHttpClientSpy).isDownloadFailed(any());
+
+ dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock);
+ dfcHttpClientSpy.close();
+
+ verify(dfcHttpClientSpy, times(1)).getServerResponse(ArgumentMatchers.eq(REMOTE_FILE));
+ verify(dfcHttpClientSpy, times(1)).processDataFromServer(any(), any(), any());
+ verify(dfcHttpClientSpy, times(1)).isDownloadFailed(any());
+ }
+
+ @Test
void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException {
String ERROR_RESPONSE = "This is unexpected message";
String REMOTE_FILE = "any";
@@ -123,7 +136,7 @@
doReturn(fis).when(dfcHttpClientSpy).getServerResponse(any());
assertThatThrownBy(() -> dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock))
- .hasMessageContaining("Error occured during datafile download: ");
+ .hasMessageContaining(ERROR_RESPONSE);
verify(dfcHttpClientSpy, times(1)).getServerResponse(REMOTE_FILE);
verify(dfcHttpClientSpy, times(1)).processFailedConnectionWithServer(any(), any());
dfcHttpClientSpy.close();
@@ -142,4 +155,16 @@
.port(PORT)
.build();
}
+
+ private ImmutableFileServerData fileServerDataWithJWTToken() throws URISyntaxException {
+ String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD;
+
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("")
+ .password("")
+ .port(PORT)
+ .queryParameters(new URIBuilder(query).getQueryParams())
+ .build();
+ }
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
index 8df91d3..168bb08 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java
@@ -16,6 +16,7 @@
package org.onap.dcaegen2.collectors.datafile.http;
+import org.apache.hc.core5.net.URIBuilder;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ConnectTimeoutException;
@@ -25,15 +26,18 @@
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import org.onap.dcaegen2.collectors.datafile.commons.FileServerData;
import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URISyntaxException;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@@ -50,6 +54,8 @@
private static final String PASSWORD = "123";
private static final String XNF_ADDRESS = "127.0.0.1";
private static final int PORT = 443;
+ private static final String JWT_PASSWORD = "thisIsThePassword";
+ private static String ACCESS_TOKEN = "access_token";
private static String remoteFile = "remoteFile";
@Mock
@@ -105,6 +111,29 @@
}
@Test
+ void dfcHttpsClient_flow_successfulCallWithJWTAndResponseProcessing() throws Exception {
+ FileServerData serverData = jWTTokenInFileServerData();
+ dfcHttpsClientSpy = spy(new DfcHttpsClient(serverData, connectionManager));
+
+ doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy)
+ .executeHttpClient(any(HttpGet.class));
+ doReturn((long)3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class));
+
+ dfcHttpsClientSpy.open();
+ dfcHttpsClientSpy.collectFile(remoteFile, localFile);
+ dfcHttpsClientSpy.close();
+
+ verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class));
+ verify(dfcHttpsClientSpy, times(1))
+ .executeHttpClient(any(HttpGet.class));
+ verify(dfcHttpsClientSpy, times(1))
+ .processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile);
+ verify(dfcHttpsClientSpy, times(1))
+ .writeFile(eq(localFile), any(InputStream.class));
+ assertFalse(serverData.toString().contains(JWT_PASSWORD));
+ }
+
+ @Test
void dfcHttpsClient_flow_failedCallUnexpectedResponseCode() throws Exception {
doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy)
.executeHttpClient(any(HttpGet.class));
@@ -112,7 +141,7 @@
dfcHttpsClientSpy.open();
- assertThrows(NonRetryableDatafileTaskException.class,
+ assertThrows(DatafileTaskException.class,
() -> dfcHttpsClientSpy.collectFile(remoteFile, localFile));
}
@@ -157,8 +186,19 @@
private ImmutableFileServerData invalidUserInFileServerData() {
return ImmutableFileServerData.builder()
.serverAddress(XNF_ADDRESS)
- .userId("demo").password("")
+ .userId(USERNAME).password("")
.port(PORT)
.build();
}
+
+ private ImmutableFileServerData jWTTokenInFileServerData() throws URISyntaxException {
+ String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD;
+
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("").password("")
+ .port(PORT)
+ .queryParameters(new URIBuilder(query).getQueryParams())
+ .build();
+ }
}
diff --git a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java
index a95bfe2..953f322 100644
--- a/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java
+++ b/datafile-app-server/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java
@@ -1,6 +1,7 @@
/*
* ============LICENSE_START======================================================================
* Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
+ * Modifications Copyright (C) 2021 Nokia. All rights reserved.
* ===============================================================================================
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@@ -16,20 +17,167 @@
package org.onap.dcaegen2.collectors.datafile.service;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
+import org.junit.jupiter.api.Test;
+import org.onap.dcaegen2.collectors.datafile.commons.ImmutableFileServerData;
+
+import java.net.URISyntaxException;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import org.junit.jupiter.api.Test;
-
class HttpUtilsTest {
+ private static final String XNF_ADDRESS = "127.0.0.1";
+ private static final int PORT = 443;
+ private static final String JWT_PASSWORD = "thisIsThePassword";
+ private static final String ACCESS_TOKEN = "access_token";
+ private static final String ANOTHER_TOKEN = "another_token";
+ private static final String ANOTHER_DATA = "another_data";
+ private static final String FRAGMENT = "thisIsTheFragment";
+ private static final String USERNAME = "bob";
+ private static final String PASSWORD = "123";
+
@Test
- public void shouldReturnSuccessfulResponse() {
- assertTrue(HttpUtils.isSuccessfulResponseCode(200));
+ void shouldReturnSuccessfulResponse() {
+ assertTrue(HttpUtils.isSuccessfulResponseCodeWithDataRouter(200));
}
@Test
- public void shouldReturnBadResponse() {
- assertFalse(HttpUtils.isSuccessfulResponseCode(404));
+ void shouldReturnBadResponse() {
+ assertFalse(HttpUtils.isSuccessfulResponseCodeWithDataRouter(404));
+ }
+
+ @Test
+ void isSingleQueryWithJWT_validToken() throws URISyntaxException {
+ assertTrue(HttpUtils.isQueryWithSingleJWT(validTokenSingleQueryData()));
+ assertTrue(HttpUtils.isQueryWithSingleJWT(validTokenDoubleQueryData()));
+ }
+
+ @Test
+ void isSingleQueryWithJWT_invalidToken() throws URISyntaxException {
+ assertFalse(HttpUtils.isQueryWithSingleJWT(validQueryNoToken()));
+ assertFalse(HttpUtils.isQueryWithSingleJWT(queryDataDoubleToken()));
+ assertFalse(HttpUtils.isQueryWithSingleJWT(null));
+ }
+
+ @Test
+ void getJWTToken_jWTTokenPresent() throws URISyntaxException {
+ assertEquals(JWT_PASSWORD, HttpUtils.getJWTToken(fileServerDataWithJWTToken()));
+ assertEquals(JWT_PASSWORD, HttpUtils.getJWTToken(fileServerDataWithJWTTokenLongQueryAndFragment()));
+ }
+
+ @Test
+ void getJWTToken_JWTTokenNotPresent() throws URISyntaxException {
+ assertEquals("", HttpUtils.getJWTToken(fileServerDataQueryWithoutToken()));
+ }
+
+ @Test
+ void prepareUri_UriWithoutPort() {
+ ImmutableFileServerData serverData = ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId(USERNAME).password(PASSWORD)
+ .build();
+ String REMOTE_FILE = "any";
+
+ String retrievedUri = HttpUtils.prepareUri("http", serverData, REMOTE_FILE, 80);
+ assertTrue(retrievedUri.startsWith("http://" + XNF_ADDRESS + ":80"));
+ }
+
+ @Test
+ void prepareUri_verifyUriWithTokenAndFragment() throws URISyntaxException {
+ String file = "/file";
+ String expected = "http://" + XNF_ADDRESS + ":" + PORT + file + "?"
+ + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"
+ + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "#" + FRAGMENT;
+ assertEquals(expected, HttpUtils.prepareUri("http", fileServerDataWithJWTTokenLongQueryAndFragment(), file, 443));
+ }
+
+ @Test
+ void prepareUri_verifyUriWithoutTokenAndWithoutFragment() throws URISyntaxException {
+ String file = "/file";
+ String expected = "http://" + XNF_ADDRESS + ":" + PORT + file;
+ assertEquals(expected, HttpUtils.prepareUri("http", fileServerDataNoTokenNoFragment(), file, 443));
+ }
+
+ private List<NameValuePair> validTokenSingleQueryData() throws URISyntaxException {
+ String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD;
+ return new URIBuilder(query).getQueryParams();
+ }
+
+ private List<NameValuePair> validTokenDoubleQueryData() throws URISyntaxException {
+ StringBuilder doubleQuery = new StringBuilder();
+ doubleQuery.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&");
+ doubleQuery.append(ACCESS_TOKEN + "=" + JWT_PASSWORD);
+ return new URIBuilder(doubleQuery.toString()).getQueryParams();
+ }
+
+ private List<NameValuePair> validQueryNoToken() throws URISyntaxException {
+ String query = "?" + ANOTHER_TOKEN + "=" + JWT_PASSWORD;
+ return new URIBuilder(query).getQueryParams();
+ }
+
+ private List<NameValuePair> queryDataDoubleToken() throws URISyntaxException {
+ StringBuilder doubleToken = new StringBuilder();
+ doubleToken.append("?" + ACCESS_TOKEN + "=" + JWT_PASSWORD + "&");
+ doubleToken.append(ACCESS_TOKEN + "=" + JWT_PASSWORD + "&");
+ doubleToken.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA);
+ return new URIBuilder(doubleToken.toString()).getQueryParams();
+ }
+
+ private ImmutableFileServerData fileServerDataWithJWTToken() throws URISyntaxException {
+ String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD;
+
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("")
+ .password("")
+ .port(PORT)
+ .queryParameters(new URIBuilder(query).getQueryParams())
+ .build();
+ }
+
+ private ImmutableFileServerData fileServerDataWithJWTTokenLongQueryAndFragment() throws URISyntaxException {
+ StringBuilder query = new StringBuilder();
+ query.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&");
+ query.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&");
+ query.append(ACCESS_TOKEN + "=" + JWT_PASSWORD + "&");
+ query.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA);
+
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("")
+ .password("")
+ .port(PORT)
+ .queryParameters(new URIBuilder(query.toString()).getQueryParams())
+ .uriRawFragment(FRAGMENT)
+ .build();
+ }
+
+ private ImmutableFileServerData fileServerDataQueryWithoutToken() throws URISyntaxException {
+ StringBuilder query = new StringBuilder();
+ query.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA);
+
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("")
+ .password("")
+ .port(PORT)
+ .queryParameters(new URIBuilder(query.toString()).getQueryParams())
+ .build();
+ }
+
+ private ImmutableFileServerData fileServerDataNoTokenNoFragment() throws URISyntaxException {
+ return ImmutableFileServerData.builder()
+ .serverAddress(XNF_ADDRESS)
+ .userId("")
+ .password("")
+ .port(PORT)
+ .queryParameters(new URIBuilder("").getQueryParams())
+ .uriRawFragment("")
+ .build();
}
}
diff --git a/pom.xml b/pom.xml
index 3a2fade..246e117 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,7 +31,7 @@
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.5.3-SNAPSHOT</version>
+ <version>1.5.4-SNAPSHOT</version>
<name>dcaegen2-collectors.datafile</name>
<description>datafile collector</description>
@@ -59,6 +59,7 @@
<commons-io.version>2.8.0</commons-io.version>
<commons-net.version>3.3</commons-net.version>
<projectreactor.version>2020.0.2</projectreactor.version>
+ <httpcomponents.core5.version>5.0.3</httpcomponents.core5.version>
<!-- LOGGING SETTINGS -->
<slf4j.version>1.7.25</slf4j.version>
@@ -189,6 +190,11 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5</artifactId>
+ <version>${httpcomponents.core5.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit-jupiter.version}</version>
diff --git a/version.properties b/version.properties
index 8e26230..8086968 100644
--- a/version.properties
+++ b/version.properties
@@ -1,6 +1,6 @@
major=1
minor=5
-patch=3
+patch=4
base_version=${major}.${minor}.${patch}
release_version=${base_version}
snapshot_version=${base_version}-SNAPSHOT