Adding & Testing method for breaking JSON Data into 'fragments'
Improving Bookstore test model
https://jira.onap.org/browse/CPS-32
Issue-ID: CPS-32
Change-Id: Ie03e03c041233aa908ab55902c1b387f96eb1c2e
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>
diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/Fragment.java b/cps-service/src/main/java/org/onap/cps/api/impl/Fragment.java
new file mode 100644
index 0000000..252b09e
--- /dev/null
+++ b/cps-service/src/main/java/org/onap/cps/api/impl/Fragment.java
@@ -0,0 +1,158 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2020 Nordix Foundation
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.api.impl;
+
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import lombok.Getter;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.model.api.DataSchemaNode;
+import org.opendaylight.yangtools.yang.model.api.Module;
+
+/**
+ * Class to store a Yang Fragment (container or list element).
+ */
+public class Fragment {
+
+ @Getter
+ private String xpath;
+
+ @Getter
+ private final Map<String, Object> attributes = new HashMap<>();
+
+ @Getter
+ private final Module module;
+
+ @Getter
+ private final Fragment parentFragment;
+
+ @Getter
+ private final Set<Fragment> childFragments = new HashSet<>(0);
+
+ private final QName[] qnames;
+
+ private Optional<Set<String>> optionalLeafListNames = Optional.empty();
+
+ /**
+ * Create a root Fragment.
+ *
+ * @param module the Yang module that encompasses this fragment
+ * @param qnames the list of qualified names that points the schema node for this fragment
+ */
+ public static Fragment createRootFragment(final Module module, final QName... qnames) {
+ return new Fragment(null, module, qnames);
+ }
+
+ /**
+ * Create a Child Fragment under a given Parent Fragment.
+ *
+ * @param parentFragment the parent (can be null for 'root' objects)
+ * @param module the Yang module that encompasses this fragment
+ * @param qnames the list of qualified names that points the schema node for this fragment
+ */
+ private Fragment(final Fragment parentFragment, final Module module, final QName... qnames) {
+ this.parentFragment = parentFragment;
+ this.module = module;
+ this.qnames = qnames;
+ }
+
+ /**
+ * Create a Child Fragment where the current Fragment is the parent.
+ *
+ * @param qnameChild The Qualified name for the child (relative to the parent)
+ * @return the child fragment
+ */
+ public Fragment createChildFragment(final QName qnameChild) {
+ final QName[] qnamesForChild = Arrays.copyOf(qnames, qnames.length + 1);
+ qnamesForChild[qnamesForChild.length - 1] = qnameChild;
+ final Fragment childFragment = new Fragment(this, module, qnamesForChild);
+ childFragments.add(childFragment);
+ return childFragment;
+ }
+
+ /**
+ * Define a leaf list by providing its name.
+ * The list is not instantiated until the first value is provided
+ *
+ * @param name the name of the leaf list
+ */
+ public void addLeafListName(final String name) {
+ if (optionalLeafListNames.isEmpty()) {
+ optionalLeafListNames = Optional.of(new HashSet<>());
+ }
+ optionalLeafListNames.get().add(name);
+ }
+
+ /**
+ * Add a leaf or leaf list value.
+ * For Leaf lists it is essential to first define the attribute is a leaf list by using addLeafListName method
+ *
+ * @param name the name of the leaf (or leaf list)
+ * @param value the value of the leaf (or element of leaf list)
+ */
+ public void addLeafValue(final String name, final Object value) {
+ if (optionalLeafListNames.isPresent() && optionalLeafListNames.get().contains(name)) {
+ addLeafListValue(name, value);
+ } else {
+ attributes.put(name, value);
+ }
+ }
+
+ private void addLeafListValue(final String name, final Object value) {
+ if (attributes.containsKey(name)) {
+ final ImmutableList<Object> oldList = (ImmutableList<Object>) attributes.get(name);
+ final List<Object> newList = new ArrayList<>(oldList);
+ newList.add(value);
+ attributes.put(name, ImmutableList.copyOf(newList));
+ } else {
+ attributes.put(name, ImmutableList.of(value));
+ }
+ }
+
+ /**
+ * Get the SchemaNodeIdentifier for this fragment.
+ *
+ * @return the SchemaNodeIdentifier
+ */
+ public String getSchemaNodeIdentifier() {
+ final StringBuilder stringBuilder = new StringBuilder();
+ for (final QName qname : qnames) {
+ stringBuilder.append(qname.getLocalName());
+ stringBuilder.append('/');
+ }
+ return stringBuilder.toString();
+ }
+
+ /**
+ * Get the Optional SchemaNode (model) for this data fragment.
+ *
+ * @return the Optional SchemaNode
+ */
+ public Optional<DataSchemaNode> getSchemaNode() {
+ return module.findDataTreeChild(qnames);
+ }
+}
diff --git a/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java b/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java
index e9757ec..0f05d7d 100644
--- a/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java
+++ b/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java
@@ -23,15 +23,26 @@
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
+import java.util.Collection;
import java.util.Iterator;
import java.util.ServiceLoader;
+import java.util.logging.Logger;
+import org.onap.cps.api.impl.Fragment;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.DataContainerNode;
+import org.opendaylight.yangtools.yang.data.api.schema.LeafSetNode;
+import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
+import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.ValueNode;
import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter;
import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactory;
import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactorySupplier;
import org.opendaylight.yangtools.yang.data.codec.gson.JsonParserStream;
import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNormalizedNodeStreamWriter;
import org.opendaylight.yangtools.yang.data.impl.schema.NormalizedNodeResult;
+import org.opendaylight.yangtools.yang.model.api.Module;
import org.opendaylight.yangtools.yang.model.api.SchemaContext;
import org.opendaylight.yangtools.yang.model.parser.api.YangParser;
import org.opendaylight.yangtools.yang.model.parser.api.YangParserException;
@@ -43,6 +54,8 @@
private static final YangParserFactory PARSER_FACTORY;
+ private static final Logger LOGGER = Logger.getLogger(YangUtils.class.getName());
+
private YangUtils() {
throw new IllegalStateException("Utility class");
}
@@ -57,14 +70,14 @@
/**
* Parse a file containing yang modules.
- * @param yangModelFile a file containing one or more yang modules
- * (please note the file has to have a .yang extension if not an exception will be thrown)
+ *
+ * @param yangModelFile a file containing one or more yang modules. The file has to have a .yang extension.
* @return a SchemaContext representing the yang model
- * @throws IOException when the system as an IO issue
+ * @throws IOException when the system as an IO issue
* @throws YangParserException when the file does not contain a valid yang structure
*/
public static SchemaContext parseYangModelFile(final File yangModelFile) throws IOException, YangParserException {
- YangTextSchemaSource yangTextSchemaSource = YangTextSchemaSource.forFile(yangModelFile);
+ final YangTextSchemaSource yangTextSchemaSource = YangTextSchemaSource.forFile(yangModelFile);
final YangParser yangParser = PARSER_FACTORY
.createParser(StatementParserMode.DEFAULT_MODE);
yangParser.addSource(yangTextSchemaSource);
@@ -73,18 +86,19 @@
/**
* Parse a file containing json data for a certain model (schemaContext).
- * @param jsonData a string containing json data for the given model
+ *
+ * @param jsonData a string containing json data for the given model
* @param schemaContext the SchemaContext for the given data
* @return the NormalizedNode representing the json data
*/
public static NormalizedNode<?, ?> parseJsonData(final String jsonData, final SchemaContext schemaContext)
throws IOException {
- JSONCodecFactory jsonCodecFactory = JSONCodecFactorySupplier.DRAFT_LHOTKA_NETMOD_YANG_JSON_02
+ final JSONCodecFactory jsonCodecFactory = JSONCodecFactorySupplier.DRAFT_LHOTKA_NETMOD_YANG_JSON_02
.getShared(schemaContext);
final NormalizedNodeResult normalizedNodeResult = new NormalizedNodeResult();
final NormalizedNodeStreamWriter normalizedNodeStreamWriter = ImmutableNormalizedNodeStreamWriter
.from(normalizedNodeResult);
- try (JsonParserStream jsonParserStream = JsonParserStream
+ try (final JsonParserStream jsonParserStream = JsonParserStream
.create(normalizedNodeStreamWriter, jsonCodecFactory)) {
final JsonReader jsonReader = new JsonReader(new StringReader(jsonData));
jsonParserStream.parse(jsonReader);
@@ -92,8 +106,69 @@
return normalizedNodeResult.getResult();
}
- public static void chopNormalizedNode(NormalizedNode<?, ?> tree) {
- //TODO Toine Siebelink, add code from proto-type (other user story)
+ /**
+ * Break a Normalized Node tree into fragments that can be stored by the persistence service.
+ *
+ * @param tree the normalized node tree
+ * @param module the module applicable for the data in the normalized node
+ * @return the 'root' Fragment for the tree contain all relevant children etc.
+ */
+ public static Fragment fragmentNormalizedNode(
+ final NormalizedNode<? extends YangInstanceIdentifier.PathArgument, ?> tree,
+ final Module module) {
+ final QName nodeType = tree.getNodeType();
+ final Fragment rootFragment = Fragment.createRootFragment(module, nodeType);
+ fragmentNormalizedNode(rootFragment, tree);
+ return rootFragment;
}
+ private static void fragmentNormalizedNode(final Fragment currentFragment,
+ final NormalizedNode normalizedNode) {
+ if (normalizedNode instanceof DataContainerNode) {
+ inspectContainer(currentFragment, (DataContainerNode) normalizedNode);
+ } else if (normalizedNode instanceof MapNode) {
+ inspectKeyedList(currentFragment, (MapNode) normalizedNode);
+ } else if (normalizedNode instanceof ValueNode) {
+ inspectLeaf(currentFragment, (ValueNode) normalizedNode);
+ } else if (normalizedNode instanceof LeafSetNode) {
+ inspectLeafList(currentFragment, (LeafSetNode) normalizedNode);
+ } else {
+ LOGGER.warning("Cannot normalize " + normalizedNode.getClass());
+ }
+ }
+
+ private static void inspectLeaf(final Fragment currentFragment,
+ final ValueNode valueNode) {
+ final Object value = valueNode.getValue();
+ currentFragment.addLeafValue(valueNode.getNodeType().getLocalName(), value);
+ }
+
+ private static void inspectLeafList(final Fragment currentFragment,
+ final LeafSetNode leafSetNode) {
+ currentFragment.addLeafListName(leafSetNode.getNodeType().getLocalName());
+ for (final NormalizedNode value : (Collection<NormalizedNode>) leafSetNode.getValue()) {
+ fragmentNormalizedNode(currentFragment, value);
+ }
+ }
+
+ private static void inspectContainer(final Fragment currentFragment,
+ final DataContainerNode dataContainerNode) {
+ final Collection<NormalizedNode> leaves = (Collection) dataContainerNode.getValue();
+ for (final NormalizedNode leaf : leaves) {
+ fragmentNormalizedNode(currentFragment, leaf);
+ }
+ }
+
+ private static void inspectKeyedList(final Fragment currentFragment,
+ final MapNode mapNode) {
+ createNodeForEachListElement(currentFragment, mapNode);
+ }
+
+ private static void createNodeForEachListElement(final Fragment currentFragment, final MapNode mapNode) {
+ final Collection<MapEntryNode> mapEntryNodes = mapNode.getValue();
+ for (final MapEntryNode mapEntryNode : mapEntryNodes) {
+ final Fragment listElementFragment = currentFragment.createChildFragment(mapNode.getNodeType());
+ fragmentNormalizedNode(listElementFragment, mapEntryNode);
+ }
+ }
}
diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy
index 709378e..2a3d5cb 100755
--- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy
+++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy
@@ -67,7 +67,7 @@
}
def assertModule(SchemaContext schemaContext){
- def optionalModule = schemaContext.findModule('bookstore', Revision.of('2020-09-15'))
+ def optionalModule = schemaContext.findModule('stores', Revision.of('2020-09-15'))
return schemaContext.modules.size() == 1 && optionalModule.isPresent()
}
@@ -167,4 +167,4 @@
expect: 'anchor name is returned by service'
objectUnderTest.createAnchor(anchorDetails) == 'dummyAnchorName'
}
-}
\ No newline at end of file
+}
diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy
index 8aabc48..6a463ad 100644
--- a/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy
+++ b/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy
@@ -19,14 +19,16 @@
package org.onap.cps.utils
-import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode
+import org.onap.cps.TestUtils
import org.opendaylight.yangtools.yang.common.QName
import org.opendaylight.yangtools.yang.common.Revision
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode
import org.opendaylight.yangtools.yang.model.parser.api.YangSyntaxErrorException
import spock.lang.Specification
import spock.lang.Unroll
class YangUtilsSpec extends Specification{
+
def 'Parsing a valid Yang Model'() {
given: 'a yang model (file)'
def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
@@ -34,14 +36,14 @@
def result = YangUtils.parseYangModelFile(file)
then: 'the result contain 1 module of the correct name and revision'
result.modules.size() == 1
- def optionalModule = result.findModule('bookstore', Revision.of('2020-09-15'))
+ def optionalModule = result.findModule('stores', Revision.of('2020-09-15'))
optionalModule.isPresent()
}
@Unroll
- def 'parsing invalid yang file (#description)'() {
+ def 'Parsing invalid yang file (#description).'() {
given: 'a file with #description'
- File file = new File(ClassLoader.getSystemClassLoader().getResource(filename).getFile());
+ File file = new File(ClassLoader.getSystemClassLoader().getResource(filename).getFile())
when: 'the file is parsed'
YangUtils.parseYangModelFile(file)
then: 'an exception is thrown'
@@ -52,29 +54,48 @@
'someOtherFile.txt' | 'no .yang extension' || IllegalArgumentException
}
- def 'Parsing a valid Json String'() {
+ def 'Parsing a valid Json String.'() {
given: 'a yang model (file)'
def jsonData = org.onap.cps.TestUtils.getResourceFileContent('bookstore.json')
and: 'a model for that data'
def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
def schemaContext = YangUtils.parseYangModelFile(file)
when: 'the json data is parsed'
- NormalizedNode<?, ?> result = YangUtils.parseJsonData(jsonData, schemaContext);
+ NormalizedNode<?, ?> result = YangUtils.parseJsonData(jsonData, schemaContext)
then: 'the result is a normalized node of the correct type'
- result.nodeType == QName.create('org:onap:ccsdk:sample','2020-09-15','bookstore')
+ result.nodeType == QName.create('org:onap:ccsdk:sample', '2020-09-15', 'bookstore')
}
- def 'Parsing an invalid Json String'() {
+ @Unroll
+ def 'Parsing invalid data: #description.'() {
given: 'a yang model (file)'
- def jsonData = '{incomplete json'
- and: 'a model'
def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
def schemaContext = YangUtils.parseYangModelFile(file)
- when: 'the invalid json is parsed'
- YangUtils.parseJsonData(jsonData, schemaContext);
- then: ' an exception is thrown'
+ when: 'invalid data is parsed'
+ YangUtils.parseJsonData(invalidJson, schemaContext)
+ then: 'an exception is thrown'
thrown(IllegalStateException)
+ where: 'the following invalid json is provided'
+ invalidJson | description
+ '{incomplete json' | 'incomplete json'
+ '{"test:bookstore": {"address": "Parnell st." }}' | 'json with un-modelled data'
}
+ def 'Breaking a Json Data Object into fragments.'() {
+ given: 'a Yang module'
+ def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
+ def schemaContext = YangUtils.parseYangModelFile(file)
+ def module = schemaContext.findModule('stores', Revision.of('2020-09-15')).get()
+ and: 'a normalized node for that model'
+ def jsonData = TestUtils.getResourceFileContent('bookstore.json')
+ def normalizedNode = YangUtils.parseJsonData(jsonData, schemaContext)
+ when: 'the json data is fragmented'
+ def result = YangUtils.fragmentNormalizedNode(normalizedNode, module)
+ then: 'the system creates a (root) fragment without a parent and 2 children (categories)'
+ result.parentFragment == null
+ result.childFragments.size() == 2
+ and: 'each child (category) has the root fragment (result) as parent and in turn as 1 child (a list of books)'
+ result.childFragments.each { it.parentFragment == result && it.childFragments.size() == 1 }
+ }
}
diff --git a/cps-service/src/test/resources/bookstore.json b/cps-service/src/test/resources/bookstore.json
index 44d5d42..d1b8d68 100644
--- a/cps-service/src/test/resources/bookstore.json
+++ b/cps-service/src/test/resources/bookstore.json
@@ -1,34 +1,52 @@
{
"test:bookstore":{
- "categories":[
+ "bookstore-name": "Chapters",
+ "categories": [
{
- "name":"web",
- "books":[
+ "code": "01",
+ "name": "SciFi",
+ "books": [
{
- "authors":[
- "Toine Siebelink","David Lang"
+ "authors": [
+ "Iain M. Banks"
],
- "lang":"en",
- "price":"123456",
- "pub_year":"2020",
- "title":"My first book"
+ "lang": "en",
+ "price": "895",
+ "pub_year": "1994",
+ "title": "Feersum Endjinn"
+ },
+ {
+ "authors": [
+ "Ursula K. Le Guin",
+ "Joe Haldeman",
+ "Orson Scott Card",
+ "david Brin",
+ "Rober Silverberg",
+ "Dan Simmons",
+ "Greg Bear"
+ ],
+ "lang": "en",
+ "price": "1099",
+ "pub_year": "1999",
+ "title": "Far Horizons"
}
]
},
{
- "name":"art",
- "books":[
+ "name": "kids",
+ "code": "02",
+ "books": [
{
- "authors":[
- "Test"
+ "authors": [
+ "Philip Pullman"
],
- "lang":"en",
- "price":"1234",
- "pub_year":"2020",
- "title":"My 2nd book"
+ "lang": "en",
+ "price": "699",
+ "pub_year": "1995",
+ "title": "The Golden Compass"
}
]
}
]
}
-}
\ No newline at end of file
+}
diff --git a/cps-service/src/test/resources/bookstore.yang b/cps-service/src/test/resources/bookstore.yang
index 01eac5f..2179fb9 100644
--- a/cps-service/src/test/resources/bookstore.yang
+++ b/cps-service/src/test/resources/bookstore.yang
@@ -1,6 +1,5 @@
-module bookstore {
+module stores {
yang-version 1.1;
-
namespace "org:onap:ccsdk:sample";
prefix book-store;
@@ -18,9 +17,17 @@
container bookstore {
+ leaf bookstore-name {
+ type string;
+ }
+
list categories {
- key name;
+ key "code";
+
+ leaf code {
+ type string;
+ }
leaf name {
type string;