ui-react/src/utils/CsvToJson.js

/*-
 * ============LICENSE_START=======================================================
 * ONAP CLAMP
 * ================================================================================
 * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
 * ================================================================================
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ============LICENSE_END============================================
 * ===================================================================
 *
 */

export default function CsvToJson(rawCsvData, delimiter, internalDelimiter, csvHeaderNames, jsonKeyNames, mandatory) {

	let printDictKeys = '';
	let result = { jsonObjArray: [], errorMessages: '' };

	// Validate that all parallel arrays passed in have same number of elements;
	// this would be a developer error.

	let checkLength = csvHeaderNames.length;

	if (checkLength !== jsonKeyNames.length || checkLength !== mandatory.length) {
		result.errorMessages = 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays parameters are not the same length';
		return result;
	}

	if (checkLength < 1) {
		result.errorMessages = 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays have no entries';
		return result;
	}

	// Make a nice string to print in the error case to tell user what is the
	//  required heaer row format

	for (let i=0; i < csvHeaderNames.length; ++i) {
		if (i === 0) {
			printDictKeys = csvHeaderNames[i];
		} else {
			printDictKeys += ',' +  csvHeaderNames[i];
		}
	}

	let dictElems = rawCsvData.split('\n');
	let numColumns = 0;
	let filteredDictElems = [];

	// The task of the following loop is to convert raw CSV rows into easily parseable
	// and streamlined versions of the rows with an internalDelimiter replacing the standard
	// comma; it is presumed (and checked) that the internalDelimiter cannot exist as a valid
	// sequence of characters in the user's data.

	// This conversion process also strips leading and trailing whitespace from each row,
	// discards empty rows, correctly interprets and removes all double quotes that programs like
	// Excel use to support user columns that contain special characters, most notably, the comma
	// delimiter. A double-quote that is contained within a double-quoted column value 
	// must appear in this raw data as a sequence of two double quotes. Furthermore, any column
	// value in the raw CSV data that does not contain a delimiter may or may not be enclosed in
	// double quotes. It is the Excel convention to not use double qoutes unless necessary, and
	// there is no reasonable way to tell Excel to surround every column value with double quotes. 
	// Any files that were directly "exported" by CLAMP itself from the Managing Dictionaries
	// capability, surround all columns with double quotes.

	for (let i = 0; i < dictElems.length; i++) {

		let oneRow = dictElems[i].trim();
		let j = 0;
		let inQuote = false
		let nextChar = undefined;
		let prevChar = null;

		
		if (oneRow === '') {
			continue; // Skip blank rows
		} else if (oneRow.indexOf(internalDelimiter) !== -1) {
			result.errorMessages += '\nRow #' + i + ' contains illegal sequence of characters (' + internalDelimiter + ')';
			break;
		} else {
			nextChar = oneRow[1];
		}

		let newStr = '';
		numColumns = 1;

		// This "while loop" performs the very meticulous task of removing double quotes that
		// are used by Excel to encase special characters as user string value data,
		// and manages to correctly identify columns that are defined with or without
		// double quotes and to process the comma delimiter correctly when encountered
		// as a user value within a column. Such a column would have to be encased in
		// double quotes; a comma found outside double quotes IS a delimiter.

		while (j < oneRow.length) {
		 	if (oneRow[j] === '"') {
		 		if (inQuote === false) {
					if (prevChar !== delimiter && prevChar !== null) {
						result.errorMessages += '\nMismatched double quotes or illegal whitespace around delimiter at row #' + (i + 1) + ' near column #' + numColumns;
						break;
					} else {
						inQuote = true;
					}
				} else {
					if (nextChar === '"') {
						newStr += '"';
						++j;
					} else if ((nextChar !== delimiter) && (nextChar !== undefined)) {
						result.errorMessages += '\nRow #' + (i + 1) + ' is badly formatted at column #' + numColumns + '. Perhaps an unescaped double quote.';
						break;
					} else if (nextChar === delimiter) {
						++numColumns;
						inQuote = false;
						newStr += internalDelimiter;
						prevChar = delimiter;
						j += 2;
						nextChar = oneRow[j+1];
						continue;
					} else {
						++numColumns;
						inQuote = false;
						break;
					}
				}
			} else {
				if (oneRow[j] === delimiter && inQuote === false) {
					newStr += internalDelimiter;
					++numColumns;
				} else {
					newStr += oneRow[j];
				}
			}
			prevChar = oneRow[j];
			++j;
			nextChar = oneRow[j+1]; // can result in undefined at the end
		}

		if (result.errorMessages === '' && inQuote !== false) {
			result.errorMessages += '\nMismatched double quotes at row #' + (i + 1);
			break;
		} else if (result.errorMessages === '' && numColumns < jsonKeyNames.length) {
			result.errorMessages += '\nNot enough columns (' + jsonKeyNames.length + ') at row #' + (i + 1);
			break;
		}

		filteredDictElems.push(newStr);
	}

	if (result.errorMessages !== '') {
		return result;
	}

	// Perform further checks on data that is now in JSON form
	if (filteredDictElems.length < 2) {
		result.errorMessages += '\nNot enough row data found in import file. Need at least a header row and one row of data';
		return result;
	}

	// Now that we have something reliably parsed into sanitized columns lets run some checks
	// and convert it all into an array of JSON objects to push to the back end if all the
	// checks pass.

	let headers = filteredDictElems[0].split(internalDelimiter);

	// check that headers are included in proper order
	for (let i=0; i < jsonKeyNames.length; ++i) {
		if (csvHeaderNames[i] !== headers[i]) {
			result.errorMessages += 'Row 1 header key at column #' + (i + 1) + ' is a mismatch. Expected row header must contain at least:\n' + printDictKeys; 
			return result;
		}
	}

	// Convert the ASCII rows of data into an array of JSON obects that omit the header
	// row which is not sent to the back end.

	for (let i = 1; i < filteredDictElems.length; i++) {
		let data = filteredDictElems[i].split(internalDelimiter);
		let obj = {};
		for (let j = 0; j < data.length && j < jsonKeyNames.length; j++) {
			let value = data[j].trim();
			if (mandatory[j] === true && value === '') {
				result.errorMessages += '\n' + csvHeaderNames[j] + ' at row #' + (i+1) + ' is empty but requires a value.';
			}
			obj[jsonKeyNames[j]] = value;
		}
		result.jsonObjArray.push(obj);
	}

	if (result.errorMessages !== '') {
		// If we have errors, return empty parse result even though some things
		// may have parsed properly. We do not want to encourage the caller
		// to think the data is good for use.
		result.jsonObjArray = [];
	}

	return result;
}