openecomp-be/tools/scripts/generate-manifest.py

#!/usr/bin/python

##############################################################################
###
### generate-manifest.py
###
### A Vendor utility to generate a valid heat zip manifest file for the AT&T onboarding. 
###
### Usage:
###
###    generate-manifest.py [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description
###
### For example:
###
###    ./generate-manifest.py --folder ./vota --name vOTA --description "HOT template to create vOTA server"
###
### Help:
### The script is doing the following:
###  1) Split the files into different types
###     a. .env files
###     b. Network files (anything containing the string network)
###     c. Volume files (anything containing the string volume)
###     d. Top level Heat files
###     e. Other types
###  2) Match env files to heat files – looking for same name ignoring suffix and extension
###  3) Match Network childs
###     a. Look for Top level heats which name is a substring of the name of the Network heat name.
###  4) Match Volume childs
###     a. Look for Top level heats which name is a substring of the name of the Volume heat name.
###  5)      Generate the JSON file from the above
###
###
### Author: Avi Ziv
### Version 1.4 for ASDC/AT&T 1.0
### Date: 13 July 2016
###
##############################################################################

# import os,sys,getopt,json,re
import os, sys, getopt, re
from collections import OrderedDict
from json import JSONEncoder
import json

VERSION = "1.4"
ENV_EXT = ".env"
SHELL_EXT = ".sh"
YAML_EXT = [".yaml", ".yml"]
# VERSION_DELIMITER_PATTERN='_v\d{*}.\d{*}'
# VERSION_DELIMITER_PATTERN='_v*.*'
#v1.0
VERSION_DELIMITER_PATTERN = '_v\d+.\d+'
#07_12_2016
VERSION_DELIMITER_PATTERN2 = '_\d+-\d+-\d+'

# types
HEAT = "HEAT"
HEAT_BASE = "HEAT_BASE"
HEAT_NET = "HEAT_NET"
HEAT_VOL = "HEAT_VOL"
HEAT_ENV = "HEAT_ENV"
SHELL = "SHELL"
OTHER = "OTHER"

globalVolumeVal = "VOLUME"
globalNetworkVal = "NETWORK"
globalBaseVal = "BASE"


def version():
    return VERSION


def usage():
    print ('usage: ' + sys.argv[0] + ' [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description' )


def header():
    print ("\nASDC Vendor manifest file generator, version " + version() + "\n")


def getEnvVariables(value, defaultValue):
    try:
        eVal = os.environ[value]
        return eVal
    except KeyError:
        print ("Missing ${" + value + "} envirunment variable. Using default value: " + defaultValue)
    return defaultValue


def getF(listFiles):
    print ("Analyzing files ...")
    foundABase = False
    files = listFiles
    jsons = {}
    lOfEnvs = {}
    lOfVolumes = {}
    lOfNetworks = {}
    lOfHeats = {}
    lOfShels = {}
    lOfArtifacts = {}

    for f in files:
        fullFilename = f[1]
        fObj = ManifestFileInfo(fullFilename)
        if fObj.isEnv():
            lOfEnvs[fObj.file_name] = fObj
        elif fObj.isShell():
            lOfShels[fObj.file_name] = fObj
        elif fObj.isVolume():
            lOfVolumes[fObj.file_name] = fObj
        elif fObj.isNetwork():
            lOfNetworks[fObj.file_name] = fObj
        elif (fObj.isYaml() and not fObj.isBase()):
            lOfHeats[fObj.file_name] = fObj
        elif fObj.isArtifact():
            lOfArtifacts[fObj.file_name] = fObj
        elif (fObj.isBase() and fObj.isYaml()):
            foundABase = True
            lOfHeats[fObj.file_name] = fObj

    jsons['heats'] = lOfHeats
    jsons['envs'] = lOfEnvs
    jsons['shells'] = lOfShels
    jsons['volumes'] = lOfVolumes
    jsons['networks'] = lOfNetworks
    jsons['artifacts'] = lOfArtifacts

    if not foundABase:
        print (">>> Warning: No Base was found")
    return jsons

def loadFilesToList(folder):
    print ("Analyzing files in folder: << " + folder + " >>")
    files = os.listdir(folder)
    listOfFiles = []
    for f in files:
        if os.path.isdir(os.path.join(folder, f)):
            ConsoleLogger.warning("Sub folders are ignored by this script, you may want to remove it before archiving")
            continue

        filename, file_extension = os.path.splitext(f)
        if filename == 'MANIFEST':
            ConsoleLogger.warning("Your folder already contains a manifest file that will be overridden")
            continue
        listOfFiles.append([filename, f])
    return listOfFiles


def make(files):
    flist = []
    dEnvs = {}
    dEnvs = files['envs']
    dHeats = files['heats']
    dNetworks = files['networks']
    dVolumes = files['volumes']
    dArtifacts = files['artifacts']
    dShells = files['shells']

    env_items = dEnvs.items()
    for fileItem in env_items:
        env_name = fileItem[1].file_name
        env_base = fileItem[1].base_file_name
        if env_name in dHeats:
            dHeats[env_name].add(fileItem[1])
            continue

        if env_name in dNetworks.items():
            dNetworks[env_name].add(fileItem[1])
            continue

        if env_name in dVolumes.items():
            dVolumes[env_name[0]].add(env_name[1])
            continue

        for fName in dHeats:
            heat_base = dHeats[fName].base_file_name
            if env_base in heat_base:
                dHeats[fName].add(dEnvs[env_name])
                break
        else:
            for fName in dNetworks:
                net_base = dNetworks[fName].base_file_name
                if env_base in net_base:
                    dNetworks[fName].add(dEnvs[env_name])
                    break
            else:
                for fName in dVolumes:
                    vol_base = dVolumes[fName].base_file_name
                    if env_base in vol_base:
                        dVolumes[fName].add(dEnvs[env_name])
                        break

                else:
                    flist.append(dEnvs[env_name])

    for fName in dVolumes:
        vol_base = dVolumes[fName].base_file_name
        for hfName in dHeats:
            heat_base = dHeats[hfName].base_file_name
            if heat_base in vol_base:
                dHeats[hfName].add(dVolumes[fName])
                break
        else:
            flist.append(dVolumes[fName])

    for fName in dNetworks:
        net_base = dNetworks[fName].base_file_name
        for hfName in dHeats:
            heat_base = dHeats[hfName].base_file_name
            if heat_base in net_base:
                dHeats[hfName].add(dNetworks[fName])
                break
        else:
            flist.append(dNetworks[fName])

    for fName in dHeats:
        flist.append(dHeats[fName])
    for fName in dShells:
        flist.append(dShells[fName])
    for fName in dArtifacts:
        flist.append(dArtifacts[fName])

    print ("\n------------------------------------------------------------\n")
    return flist


def generate(folder, name, description):
    print ("Checking envirunment variables ...")
    global globalVolumeVal
    globalVolumeVal = getEnvVariables("VOLUME", globalVolumeVal)

    global globalNetworkVal
    globalNetworkVal = getEnvVariables("NETWORK", globalNetworkVal)

    global globalBaseVal
    globalBaseVal = getEnvVariables("BASE", globalBaseVal)

    YamlTabCleaner(folder).cleanYamlTabs()

    print ("Generating manifest file ...")
    jsons = getF(loadFilesToList(folder))

    lFiles = make(jsons)
    manifest = Manifest(name, description, '1.0', lFiles)
    output_json = json.dumps(manifest, default=jdefault, indent=4, sort_keys=False)

    f = open(os.path.join(folder, 'MANIFEST.json'), 'w')
    f.write(output_json)
    print("MANIFEST file created")


################

def jdefault(obj):
    if hasattr(obj, '__json__'):
        return obj.__json__()
    else:
        return obj.__dict__


class ManifestFileInfo(object):
    def __init__(self, filename):
        self.name = filename
        self.base = 'false'
        self.data = []
        self.file_name, self.file_extension = os.path.splitext(filename)
        self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN, '', self.file_name)
        self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN2, '', self.base_file_name)

        if self.isEnv():
            self.heat_type = Types.ENV
        elif self.isShell():
            self.heat_type = Types.SHELL
        elif self.isVolume():
            self.heat_type = Types.VOL
        elif self.isNetwork():
            self.heat_type = Types.NET
        elif self.isYaml() and not self.isBase():
            self.heat_type = Types.HEAT
        elif self.isArtifact():
            self.heat_type = Types.OTHER
        elif (self.isBase() and self.isYaml()):
            self.heat_type = Types.HEAT
            self.base = 'true'

    def set(self, data):
        self.data = data

    def add(self, item):
        self.data.append(item)

    def isYaml(self):
        return any(val in self.file_extension.lower() for val in YAML_EXT)

    def isEnv(self):
        return self.file_extension.lower() == ENV_EXT.lower()

    def isShell(self):
        return self.file_extension.lower() == SHELL_EXT.lower()

    def isVolume(self):
        res = globalVolumeVal.lower() in self.file_name.lower()
        return res

    def isNetwork(self):
        res = globalNetworkVal.lower() in self.file_name.lower()
        return res

    def isBase(self):
        res = globalBaseVal.lower() in self.file_name.lower()
        return res

    def isArtifact(self):
        return (not self.isBase() and not self.isVolume() and not self.isNetwork() and not self.isEnv())

    def isHEAT(self):
        return ((self.heat_type == Types.HEAT) | (self.heat_type == Types.BASE) | (self.heat_type == Types.NET) | (
            self.heat_type == Types.VOL))

    def __json__(self):
        dict = OrderedDict(
            [('file', self.name), ('type', self.heat_type)])
        if self.isHEAT():
            dict['isBase'] = self.base
            if self.data != []:
                dict['data'] = self.data

        return dict


class Manifest(object):
    def __init__(self, name, description, version, data):
        self.name = name
        self.description = description
        self.version = version
        self.data = data

    def add(self, data):
        self.data.append(data)

    def __json__(self):
        return OrderedDict([('name', self.name), ('description', self.description), ('data', self.data)])


class YamlTabCleaner(object):
    def __init__(self, folder):
        self.folder = folder

    def replaceTabs(self, sourceFile, targetFile):
        with open(sourceFile, "rt") as fin:
            if '\t' in fin.read():
                print("\'tab\' character was found in the file: " + sourceFile + "\na clean version of the file can be found under \'clean\' folder")
                target = os.path.dirname(targetFile)
                if not os.path.exists(target):
                    os.makedirs(target)
                fin.seek(0)
                with open(targetFile, "wt") as fout:
                    for line in fin:
                        fout.write(line.replace('\t', ' '))

    def cleanYamlTabs(self):
        included_extenstions = ['yml', 'yaml']
        files = [fn for fn in os.listdir(self.folder)
                 if any(fn.endswith(ext) for ext in included_extenstions)]
        target = os.path.join(self.folder, "clean")
        for file in files:
            self.replaceTabs(os.path.join(self.folder, file), os.path.join(target, file))

class ConsoleLogger(object):
    @classmethod
    def error(cls, message):
        print(">>> Error: " + message)

    @classmethod
    def warning(cls, message):
        print(">>> Warning: " + message)


    @classmethod
    def info(cls, message):
        print(">>> Info: " + message)


def enum(**named_values):
    return type('Enum', (), named_values)


################

def main(argv):
    action = ''
    folderName = '.'
    name = ''
    description = ''
    version = ''

    try:
        opts, args = getopt.getopt(argv, "h:f:n:d", ["folder=", "name=", "description=", ])
    except getopt.GetoptError as err:
        # print help information and exit:
        print ('>>>>' + str(err))
        usage()
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            usage()
            sys.exit()
        elif opt in ('-f', '--folder'):
            action = 'generate'
            if not arg:
                print ("Error: missing heat files directory")
                usage()
                sys.exit(2)
            else:
                folderName = arg
        elif opt in ('-n', '--name'):
            name = arg
        elif opt in ('-d', '--description'):
            description = arg
        else:
            usage()

    if action == 'generate':
        generate(folderName, name, description)
        sys.exit()
    else:
        usage()


if __name__ == "__main__":
    header()
    Types = enum(HEAT='HEAT', BASE='HEAT_BASE', NET='HEAT_NET', VOL='HEAT_VOL', ENV='HEAT_ENV', SHELL='SHELL',
                 OTHER='OTHER')
    main(sys.argv[1:])