Documentation of Data file collector
Documentation of Influx Logger
Signed-off-by: PatrikBuhr <patrik.buhr@est.tech>
Issue-ID: NONRTRIC-864
Change-Id: I0acb0f72fb12638c1bed9095056d8db4a74a8dc3
diff --git a/datafilecollector/config/README b/datafilecollector/config/README
index b26be1a..fc015ae 100644
--- a/datafilecollector/config/README
+++ b/datafilecollector/config/README
@@ -28,7 +28,7 @@
## License
-Copyright (C) 2021-2023 Nordix Foundation All rights reserved.
+Copyright (C) 2022-2023 Nordix Foundation. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
diff --git a/datafilecollector/config/application.yaml b/datafilecollector/config/application.yaml
index 1b85bb9..5352f1b 100644
--- a/datafilecollector/config/application.yaml
+++ b/datafilecollector/config/application.yaml
@@ -29,11 +29,12 @@
collected-files-path: "/tmp/oran_datafile/"
# Numer of worker threads. Increased number may increase throughput, but will require more executing resources.
number-of-worker-treads: 200
- # KAFKA boostrap servers.
- # several redundant boostrap servers can be specified, separated by a comma ','.
+ # If specified, and authorization token will be loaded from file and inserted in each HTTP header when using HTTP/HTTPS.
# If the file name is empty, no authorization token is used
auth-token-file:
kafka:
+ # KAFKA boostrap servers.
+ # several redundant boostrap servers can be specified, separated by a comma ','.
bootstrap-servers: localhost:9092
# output topic
collected-file-topic: collected-file
@@ -58,11 +59,17 @@
trust-store-password-file: /opt/app/datafile/config/truststore.pass
trust-store: /opt/app/datafile/config/truststore.jks
s3:
- endpointOverride:
- accessKeyId:
- secretAccessKey:
- bucket:
- locksBucket:
+ # S3 endpoint
+ endpointOverride: http://localhost:9000
+ # S3 access key ID
+ accessKeyId: miniostorage
+ # S3 password
+ secretAccessKey: secret
+ # The bucket to store collected files
+ bucket: ropfiles
+ # The bucket where DFC keeps lockfiles. This is to avoid that the same file is transfered more than once if
+ # several load sharing DFC instances are running.
+ locksBucket: dfclocks
springdoc:
show-actuator: true
- swagger-ui.disable-swagger-default-url: true
\ No newline at end of file
+ swagger-ui.disable-swagger-default-url: true
diff --git a/datafilecollector/docs/Architecture.png b/datafilecollector/docs/Architecture.png
new file mode 100644
index 0000000..4a1b8da
--- /dev/null
+++ b/datafilecollector/docs/Architecture.png
Binary files differ
diff --git a/datafilecollector/docs/Pictures.pptx b/datafilecollector/docs/Pictures.pptx
new file mode 100644
index 0000000..2debfc9
--- /dev/null
+++ b/datafilecollector/docs/Pictures.pptx
Binary files differ
diff --git a/datafilecollector/docs/_static/logo.png b/datafilecollector/docs/_static/logo.png
new file mode 100644
index 0000000..c3b6ce5
--- /dev/null
+++ b/datafilecollector/docs/_static/logo.png
Binary files differ
diff --git a/datafilecollector/docs/api-docs.rst b/datafilecollector/docs/api-docs.rst
new file mode 100644
index 0000000..fe3663e
--- /dev/null
+++ b/datafilecollector/docs/api-docs.rst
@@ -0,0 +1,24 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. Copyright (C) 2023 Nordix
+
+.. _api_docs:
+
+.. |swagger-icon| image:: ./images/swagger.png
+ :width: 40px
+
+.. |yaml-icon| image:: ./images/yaml_logo.png
+ :width: 40px
+
+
+========
+API-Docs
+========
+
+Here we describe the APIs to access the Non-RT RIC Datafile Collector.
+
+Datafile collector
+==================
+
+No public APIs
+
diff --git a/datafilecollector/docs/conf.py b/datafilecollector/docs/conf.py
new file mode 100644
index 0000000..240d10c
--- /dev/null
+++ b/datafilecollector/docs/conf.py
@@ -0,0 +1,39 @@
+# ============LICENSE_START===============================================
+# Copyright (C) 2021-2023 Nordix Foundation. All rights reserved.
+# ========================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=================================================
+#
+
+from docs_conf.conf import *
+
+#branch configuration
+
+branch = 'latest'
+
+language = 'en'
+
+linkcheck_ignore = [
+ 'http://localhost.*',
+ 'http://127.0.0.1.*',
+ 'https://gerrit.o-ran-sc.org.*',
+ './pm-producer-api.html', #Generated file that doesn't exist at link check.
+]
+
+extensions = ['sphinx.ext.intersphinx',]
+
+
+#intershpinx mapping with other projects
+intersphinx_mapping = {}
+
+intersphinx_mapping['nonrtric'] = ('https://docs.o-ran-sc.org/projects/o-ran-sc-nonrtric/en/%s' % branch, None)
diff --git a/datafilecollector/docs/conf.yaml b/datafilecollector/docs/conf.yaml
new file mode 100644
index 0000000..8f24fdb
--- /dev/null
+++ b/datafilecollector/docs/conf.yaml
@@ -0,0 +1,3 @@
+---
+project_cfg: oran
+project: nonrtric-plt-pmproducer
diff --git a/datafilecollector/docs/developer-guide.rst b/datafilecollector/docs/developer-guide.rst
new file mode 100644
index 0000000..38fa9dc
--- /dev/null
+++ b/datafilecollector/docs/developer-guide.rst
@@ -0,0 +1,29 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. SPDX-License-Identifier: CC-BY-4.0
+.. Copyright (C) 2023 Nordix
+
+Developer Guide
+===============
+
+This document provides a quickstart for developers of the Non-RT RIC Data File Collector.
+
+Additional developer guides are available on the `O-RAN SC NONRTRIC Developer wiki <https://wiki.o-ran-sc.org/display/RICNR/Release+E>`_.
+
+Data File Collector
+-------------------
+
+The DFC is configured via the application.yaml file.
+
+Example: ":download:`link <../config/application.yaml>`"
+
+Kubernetes deployment
+=====================
+
+Non-RT RIC can be also deployed in a Kubernetes cluster, `it/dep repository <https://gerrit.o-ran-sc.org/r/admin/repos/it/dep>`_.
+hosts deployment and integration artifacts. Instructions and helm charts to deploy the Non-RT-RIC functions in the
+OSC NONRTRIC integrated test environment can be found in the *./nonrtric* directory.
+
+For more information on installation of NonRT-RIC in Kubernetes, see `Deploy NONRTRIC in Kubernetes <https://wiki.o-ran-sc.org/display/RICNR/Deploy+NONRTRIC+in+Kubernetes>`_.
+
+For more information see `Integration and Testing documentation on the O-RAN-SC wiki <https://docs.o-ran-sc.org/projects/o-ran-sc-it-dep/en/latest/index.html>`_.
+
diff --git a/datafilecollector/docs/favicon.ico b/datafilecollector/docs/favicon.ico
new file mode 100644
index 0000000..00b0fd0
--- /dev/null
+++ b/datafilecollector/docs/favicon.ico
Binary files differ
diff --git a/datafilecollector/docs/images/swagger.png b/datafilecollector/docs/images/swagger.png
new file mode 100644
index 0000000..f5a9e0c
--- /dev/null
+++ b/datafilecollector/docs/images/swagger.png
Binary files differ
diff --git a/datafilecollector/docs/images/yaml_logo.png b/datafilecollector/docs/images/yaml_logo.png
new file mode 100644
index 0000000..0492eb4
--- /dev/null
+++ b/datafilecollector/docs/images/yaml_logo.png
Binary files differ
diff --git a/datafilecollector/docs/index.rst b/datafilecollector/docs/index.rst
new file mode 100644
index 0000000..a0ef6fc
--- /dev/null
+++ b/datafilecollector/docs/index.rst
@@ -0,0 +1,14 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. SPDX-License-Identifier: CC-BY-4.0
+.. Copyright (C) 2023 Nordix
+
+Non-RT RIC PM Producer
+======================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+ ./overview.rst
+ ./developer-guide.rst
+ ./api-docs.rst
diff --git a/datafilecollector/docs/overview.rst b/datafilecollector/docs/overview.rst
new file mode 100644
index 0000000..6cf45a3
--- /dev/null
+++ b/datafilecollector/docs/overview.rst
@@ -0,0 +1,117 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. SPDX-License-Identifier: CC-BY-4.0
+.. Copyright (C) 2023 Nordix
+
+
+Data File Collector
+~~~~~~~~~~~~~~~~~~~
+
+************
+Introduction
+************
+
+The task of the Data File Collector is to collect files from traffical nodes in the RAN.
+The main use case is:
+
+* The DFC received a File Ready VES event from a Kafka topic. This contains a list of all available files.
+
+* The DFC fetches files that are not already fectched from the traffical node. This is done using one of the supported file transfer protocols.
+
+* Each file is stored in an S3 Object Store bucket or in the file system (in a persistent volume).
+
+* For each stored file, a file publish message is sent to a Kafka topic for further processing.
+
+Supported file transfer protocols are:
+
+* SFTP
+
+* FTPES
+
+* HTTP
+
+* HTTPS
+
+
+The service is implemented in Java Spring Boot.
+
+.. image:: ./Architecture.png
+ :width: 1000pt
+
+This product is a part of :doc:`NONRTRIC <nonrtric:index>`.
+
+**************************
+Input File Ready VES Event
+**************************
+
+Here follows an example of the expected input object:
+
+.. code-block:: javascript
+
+ {
+ "event":{
+ "commonEventHeader":{
+ "sequence":0,
+ "eventName":"Noti_RnNode-Ericsson_FileReady",
+ "sourceName":"5GRAN_DU",
+ "lastEpochMicrosec":151983,
+ "startEpochMicrosec":15198378,
+ "timeZoneOffset":"UTC+05:00",
+ "changeIdentifier":"PM_MEAS_FILES"
+ },
+ "notificationFields":{
+ "notificationFieldsVersion":"notificationFieldsVersion",
+ "changeType":"FileReady",
+ "changeIdentifier":"PM_MEAS_FILES",
+ "arrayOfNamedHashMap":[
+ {
+ "name":"A20220418.1900-1915_seliitdus00487.xml",
+ "hashMap":{
+ "fileFormatType":"org.3GPP.32.435#measCollec",
+ "location":"https://launchpad.net/ubuntu/+source/perf-tools-unstable/1.0+git7ffb3fd-1ubuntu1/+build/13630748/+files/perf-tools-unstable_1.0+git7ffb3fd-1ubuntu1_all.deb",
+ "fileFormatVersion":"V10",
+ "compression":"gzip"
+ }
+ }
+ ]
+ }
+ }
+ }
+
+
+***************************
+Output File Publish Message
+***************************
+
+
+Below follows an example of an output File Publish Message. .
+
+.. code-block:: javascript
+
+ {
+ "productName":"RnNode",
+ "vendorName":"Ericsson",
+ "lastEpochMicrosec":151983,
+ "sourceName":"5GRAN_DU",
+ "startEpochMicrosec":15198378,
+ "timeZoneOffset":"UTC+05:00",
+ "compression":"gzip",
+ "fileFormatType":"org.3GPP.32.435#measCollec",
+ "fileFormatVersion":"V10",
+ "name":"5GRAN_DU/A20220418.1900-1915_seliitdus00487.xml",
+ "changeIdentifier":"PM_MEAS_FILES",
+ "objectStoreBucket":"ropfiles"
+ }
+
+
+*************
+Configuration
+*************
+
+The DFC is configured via its application.yaml
+
+
+An example application.yaml configuration file: ":download:`link <../config/application.yaml>`"
+
+
+
+
diff --git a/datafilecollector/docs/requirements-docs.txt b/datafilecollector/docs/requirements-docs.txt
new file mode 100644
index 0000000..692a79f
--- /dev/null
+++ b/datafilecollector/docs/requirements-docs.txt
@@ -0,0 +1,12 @@
+tox
+Sphinx
+doc8
+docutils
+setuptools
+six
+sphinx_rtd_theme
+sphinxcontrib-needs
+sphinxcontrib-swaggerdoc
+sphinx_bootstrap_theme
+sphinxcontrib-redoc
+lfdocs-conf
\ No newline at end of file
diff --git a/datafilecollector/tox.ini b/datafilecollector/tox.ini
new file mode 100644
index 0000000..d47de5e
--- /dev/null
+++ b/datafilecollector/tox.ini
@@ -0,0 +1,37 @@
+# ==================================================================================
+# Copyright (c) 2023 Nordix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==================================================================================
+
+# documentation only
+[tox]
+minversion = 2.0
+envlist =
+ docs,
+ docs-linkcheck,
+skipsdist = true
+
+[testenv:docs]
+basepython = python3
+deps = -r{toxinidir}/docs/requirements-docs.txt
+
+commands =
+ sphinx-build -W -b html -n -d {envtmpdir}/docs/doctrees ./docs/ {toxinidir}/docs/_build/html
+ echo "Generated docs available in {toxinidir}/docs/_build/html"
+whitelist_externals = echo
+
+[testenv:docs-linkcheck]
+basepython = python3
+deps = -r{toxinidir}/docs/requirements-docs.txt
+commands = sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/linkcheck
