push addional code
Change-Id: Ia427bb3460cda3a896f8faced2de69eaf3807b74
Signed-off-by: Michael Lando <ml636r@att.com>
diff --git a/openecomp-be/tools/scripts/generate-manifest.py b/openecomp-be/tools/scripts/generate-manifest.py
new file mode 100644
index 0000000..87968e0
--- /dev/null
+++ b/openecomp-be/tools/scripts/generate-manifest.py
@@ -0,0 +1,435 @@
+#!/usr/bin/python
+
+##############################################################################
+###
+### generate-manifest.py
+###
+### A Vendor utility to generate a valid heat zip manifest file for the AT&T onboarding.
+###
+### Usage:
+###
+### generate-manifest.py [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description
+###
+### For example:
+###
+### ./generate-manifest.py --folder ./vota --name vOTA --description "HOT template to create vOTA server"
+###
+### Help:
+### The script is doing the following:
+### 1) Split the files into different types
+### a. .env files
+### b. Network files (anything containing the string network)
+### c. Volume files (anything containing the string volume)
+### d. Top level Heat files
+### e. Other types
+### 2) Match env files to heat files – looking for same name ignoring suffix and extension
+### 3) Match Network childs
+### a. Look for Top level heats which name is a substring of the name of the Network heat name.
+### 4) Match Volume childs
+### a. Look for Top level heats which name is a substring of the name of the Volume heat name.
+### 5) Generate the JSON file from the above
+###
+###
+### Author: Avi Ziv
+### Version 1.4 for OPENECOMP 1.0
+### Date: 13 July 2016 (c) OPENECOMP
+###
+##############################################################################
+
+# import os,sys,getopt,json,re
+import os, sys, getopt, re
+from collections import OrderedDict
+from json import JSONEncoder
+import json
+
+VERSION = "1.4"
+ENV_EXT = ".env"
+SHELL_EXT = ".sh"
+YAML_EXT = [".yaml", ".yml"]
+# VERSION_DELIMITER_PATTERN='_v\d{*}.\d{*}'
+# VERSION_DELIMITER_PATTERN='_v*.*'
+#v1.0
+VERSION_DELIMITER_PATTERN = '_v\d+.\d+'
+#07_12_2016
+VERSION_DELIMITER_PATTERN2 = '_\d+-\d+-\d+'
+
+# types
+HEAT = "HEAT"
+HEAT_BASE = "HEAT_BASE"
+HEAT_NET = "HEAT_NET"
+HEAT_VOL = "HEAT_VOL"
+HEAT_ENV = "HEAT_ENV"
+SHELL = "SHELL"
+OTHER = "OTHER"
+
+globalVolumeVal = "VOLUME"
+globalNetworkVal = "NETWORK"
+globalBaseVal = "BASE"
+
+
+def version():
+ return VERSION
+
+
+def usage():
+ print ('usage: ' + sys.argv[0] + ' [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description' )
+
+
+def header():
+ print ("\nASDC Vendor manifest file generator, version " + version() + "\n")
+
+
+def getEnvVariables(value, defaultValue):
+ try:
+ eVal = os.environ[value]
+ return eVal
+ except KeyError:
+ print ("Missing ${" + value + "} envirunment variable. Using default value: " + defaultValue)
+ return defaultValue
+
+
+def getF(listFiles):
+ print ("Analyzing files ...")
+ foundABase = False
+ files = listFiles
+ jsons = {}
+ lOfEnvs = {}
+ lOfVolumes = {}
+ lOfNetworks = {}
+ lOfHeats = {}
+ lOfShels = {}
+ lOfArtifacts = {}
+
+ for f in files:
+ fullFilename = f[1]
+ fObj = ManifestFileInfo(fullFilename)
+ if fObj.isEnv():
+ lOfEnvs[fObj.file_name] = fObj
+ elif fObj.isShell():
+ lOfShels[fObj.file_name] = fObj
+ elif fObj.isVolume():
+ lOfVolumes[fObj.file_name] = fObj
+ elif fObj.isNetwork():
+ lOfNetworks[fObj.file_name] = fObj
+ elif (fObj.isYaml() and not fObj.isBase()):
+ lOfHeats[fObj.file_name] = fObj
+ elif fObj.isArtifact():
+ lOfArtifacts[fObj.file_name] = fObj
+ elif (fObj.isBase() and fObj.isYaml()):
+ foundABase = True
+ lOfHeats[fObj.file_name] = fObj
+
+ jsons['heats'] = lOfHeats
+ jsons['envs'] = lOfEnvs
+ jsons['shells'] = lOfShels
+ jsons['volumes'] = lOfVolumes
+ jsons['networks'] = lOfNetworks
+ jsons['artifacts'] = lOfArtifacts
+
+ if not foundABase:
+ print (">>> Warning: No Base was found")
+ return jsons
+
+def loadFilesToList(folder):
+ print ("Analyzing files in folder: << " + folder + " >>")
+ files = os.listdir(folder)
+ listOfFiles = []
+ for f in files:
+ if os.path.isdir(os.path.join(folder, f)):
+ ConsoleLogger.warning("Sub folders are ignored by this script, you may want to remove it before archiving")
+ continue
+
+ filename, file_extension = os.path.splitext(f)
+ if filename == 'MANIFEST':
+ ConsoleLogger.warning("Your folder already contains a manifest file that will be overridden")
+ continue
+ listOfFiles.append([filename, f])
+ return listOfFiles
+
+
+def make(files):
+ flist = []
+ dEnvs = {}
+ dEnvs = files['envs']
+ dHeats = files['heats']
+ dNetworks = files['networks']
+ dVolumes = files['volumes']
+ dArtifacts = files['artifacts']
+ dShells = files['shells']
+
+ env_items = dEnvs.items()
+ for fileItem in env_items:
+ env_name = fileItem[1].file_name
+ env_base = fileItem[1].base_file_name
+ if env_name in dHeats:
+ dHeats[env_name].add(fileItem[1])
+ continue
+
+ if env_name in dNetworks.items():
+ dNetworks[env_name].add(fileItem[1])
+ continue
+
+ if env_name in dVolumes.items():
+ dVolumes[env_name[0]].add(env_name[1])
+ continue
+
+ for fName in dHeats:
+ heat_base = dHeats[fName].base_file_name
+ if env_base in heat_base:
+ dHeats[fName].add(dEnvs[env_name])
+ break
+ else:
+ for fName in dNetworks:
+ net_base = dNetworks[fName].base_file_name
+ if env_base in net_base:
+ dNetworks[fName].add(dEnvs[env_name])
+ break
+ else:
+ for fName in dVolumes:
+ vol_base = dVolumes[fName].base_file_name
+ if env_base in vol_base:
+ dVolumes[fName].add(dEnvs[env_name])
+ break
+
+ else:
+ flist.append(dEnvs[env_name])
+
+ for fName in dVolumes:
+ vol_base = dVolumes[fName].base_file_name
+ for hfName in dHeats:
+ heat_base = dHeats[hfName].base_file_name
+ if heat_base in vol_base:
+ dHeats[hfName].add(dVolumes[fName])
+ break
+ else:
+ flist.append(dVolumes[fName])
+
+ for fName in dNetworks:
+ net_base = dNetworks[fName].base_file_name
+ for hfName in dHeats:
+ heat_base = dHeats[hfName].base_file_name
+ if heat_base in net_base:
+ dHeats[hfName].add(dNetworks[fName])
+ break
+ else:
+ flist.append(dNetworks[fName])
+
+ for fName in dHeats:
+ flist.append(dHeats[fName])
+ for fName in dShells:
+ flist.append(dShells[fName])
+ for fName in dArtifacts:
+ flist.append(dArtifacts[fName])
+
+ print ("\n------------------------------------------------------------\n")
+ return flist
+
+
+def generate(folder, name, description):
+ print ("Checking envirunment variables ...")
+ global globalVolumeVal
+ globalVolumeVal = getEnvVariables("VOLUME", globalVolumeVal)
+
+ global globalNetworkVal
+ globalNetworkVal = getEnvVariables("NETWORK", globalNetworkVal)
+
+ global globalBaseVal
+ globalBaseVal = getEnvVariables("BASE", globalBaseVal)
+
+ YamlTabCleaner(folder).cleanYamlTabs()
+
+ print ("Generating manifest file ...")
+ jsons = getF(loadFilesToList(folder))
+
+ lFiles = make(jsons)
+ manifest = Manifest(name, description, '1.0', lFiles)
+ output_json = json.dumps(manifest, default=jdefault, indent=4, sort_keys=False)
+
+ f = open(os.path.join(folder, 'MANIFEST.json'), 'w')
+ f.write(output_json)
+ print("MANIFEST file created")
+
+
+################
+
+def jdefault(obj):
+ if hasattr(obj, '__json__'):
+ return obj.__json__()
+ else:
+ return obj.__dict__
+
+
+class ManifestFileInfo(object):
+ def __init__(self, filename):
+ self.name = filename
+ self.base = 'false'
+ self.data = []
+ self.file_name, self.file_extension = os.path.splitext(filename)
+ self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN, '', self.file_name)
+ self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN2, '', self.base_file_name)
+
+ if self.isEnv():
+ self.heat_type = Types.ENV
+ elif self.isShell():
+ self.heat_type = Types.SHELL
+ elif self.isVolume():
+ self.heat_type = Types.VOL
+ elif self.isNetwork():
+ self.heat_type = Types.NET
+ elif self.isYaml() and not self.isBase():
+ self.heat_type = Types.HEAT
+ elif self.isArtifact():
+ self.heat_type = Types.OTHER
+ elif (self.isBase() and self.isYaml()):
+ self.heat_type = Types.HEAT
+ self.base = 'true'
+
+ def set(self, data):
+ self.data = data
+
+ def add(self, item):
+ self.data.append(item)
+
+ def isYaml(self):
+ return any(val in self.file_extension.lower() for val in YAML_EXT)
+
+ def isEnv(self):
+ return self.file_extension.lower() == ENV_EXT.lower()
+
+ def isShell(self):
+ return self.file_extension.lower() == SHELL_EXT.lower()
+
+ def isVolume(self):
+ res = globalVolumeVal.lower() in self.file_name.lower()
+ return res
+
+ def isNetwork(self):
+ res = globalNetworkVal.lower() in self.file_name.lower()
+ return res
+
+ def isBase(self):
+ res = globalBaseVal.lower() in self.file_name.lower()
+ return res
+
+ def isArtifact(self):
+ return (not self.isBase() and not self.isVolume() and not self.isNetwork() and not self.isEnv())
+
+ def isHEAT(self):
+ return ((self.heat_type == Types.HEAT) | (self.heat_type == Types.BASE) | (self.heat_type == Types.NET) | (
+ self.heat_type == Types.VOL))
+
+ def __json__(self):
+ dict = OrderedDict(
+ [('file', self.name), ('type', self.heat_type)])
+ if self.isHEAT():
+ dict['isBase'] = self.base
+ if self.data != []:
+ dict['data'] = self.data
+
+ return dict
+
+
+class Manifest(object):
+ def __init__(self, name, description, version, data):
+ self.name = name
+ self.description = description
+ self.version = version
+ self.data = data
+
+ def add(self, data):
+ self.data.append(data)
+
+ def __json__(self):
+ return OrderedDict([('name', self.name), ('description', self.description), ('data', self.data)])
+
+
+class YamlTabCleaner(object):
+ def __init__(self, folder):
+ self.folder = folder
+
+ def replaceTabs(self, sourceFile, targetFile):
+ with open(sourceFile, "rt") as fin:
+ if '\t' in fin.read():
+ print("\'tab\' character was found in the file: " + sourceFile + "\na clean version of the file can be found under \'clean\' folder")
+ target = os.path.dirname(targetFile)
+ if not os.path.exists(target):
+ os.makedirs(target)
+ fin.seek(0)
+ with open(targetFile, "wt") as fout:
+ for line in fin:
+ fout.write(line.replace('\t', ' '))
+
+ def cleanYamlTabs(self):
+ included_extenstions = ['yml', 'yaml']
+ files = [fn for fn in os.listdir(self.folder)
+ if any(fn.endswith(ext) for ext in included_extenstions)]
+ target = os.path.join(self.folder, "clean")
+ for file in files:
+ self.replaceTabs(os.path.join(self.folder, file), os.path.join(target, file))
+
+class ConsoleLogger(object):
+ @classmethod
+ def error(cls, message):
+ print(">>> Error: " + message)
+
+ @classmethod
+ def warning(cls, message):
+ print(">>> Warning: " + message)
+
+
+ @classmethod
+ def info(cls, message):
+ print(">>> Info: " + message)
+
+
+def enum(**named_values):
+ return type('Enum', (), named_values)
+
+
+################
+
+def main(argv):
+ action = ''
+ folderName = '.'
+ name = ''
+ description = ''
+ version = ''
+
+ try:
+ opts, args = getopt.getopt(argv, "h:f:n:d", ["folder=", "name=", "description=", ])
+ except getopt.GetoptError as err:
+ # print help information and exit:
+ print ('>>>>' + str(err))
+ usage()
+ sys.exit(2)
+ for opt, arg in opts:
+ if opt == '-h':
+ usage()
+ sys.exit()
+ elif opt in ('-f', '--folder'):
+ action = 'generate'
+ if not arg:
+ print ("Error: missing heat files directory")
+ usage()
+ sys.exit(2)
+ else:
+ folderName = arg
+ elif opt in ('-n', '--name'):
+ name = arg
+ elif opt in ('-d', '--description'):
+ description = arg
+ else:
+ usage()
+
+ if action == 'generate':
+ generate(folderName, name, description)
+ sys.exit()
+ else:
+ usage()
+
+
+if __name__ == "__main__":
+ header()
+ Types = enum(HEAT='HEAT', BASE='HEAT_BASE', NET='HEAT_NET', VOL='HEAT_VOL', ENV='HEAT_ENV', SHELL='SHELL',
+ OTHER='OTHER')
+ main(sys.argv[1:])