build/build_nexus_blob.sh

#! /usr/bin/env bash

#   COPYRIGHT NOTICE STARTS HERE
#
#   Copyright 2018-2019 © Samsung Electronics Co., Ltd.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#   COPYRIGHT NOTICE ENDS HERE

### This script prepares Nexus repositories data blobs for ONAP

## The script requires following dependencies are installed: nodejs, jq, docker, twine, expect
## All required resources are expected in the upper directory created during
## download procedure as DATA_DIR or in the directory given as --input-directory
## All lists used must be in project data_lists directory or in the directory given
## as --resource-list-directory

# Fail fast settings
set -e

TIMESTAMP="date +'%Y-%m-%d_%H-%M-%S'"
SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log"

# Log everything
exec &> >(tee -a "${SCRIPT_LOG}")

# Nexus repository location
NEXUS_DOMAIN="nexus"
NEXUS_PORT="8081"
NEXUS_DOCKER_PORT="8082"
NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/"
PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/"
DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}"
DEFAULT_REGISTRY="docker.io"

# Nexus repository credentials
NEXUS_USERNAME=admin
NEXUS_PASSWORD=admin123
NEXUS_EMAIL=admin@example.org

# Setting paths
LOCAL_PATH="$(readlink -f $(dirname ${0}))"

# Defaults
DOCKER_LOAD="false"
DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
LISTS_DIR="${LOCAL_PATH}/data_lists"

# Required dependencies
COMMANDS=(jq docker expect npm twine)

usage () {
    echo "   Example usage: build_nexus_blob.sh --input-directory </path/to/downloaded/files/dir> --output-directory
           </path/to/output/dir> --resource-list-directory </path/to/dir/with/resource/list> [--load-docker-images]

     -i  | --input-directory directory containing file needed to create nexus blob. The structure of this directory must organized as described in build guide
     -ld | --load-docker-images load docker images from stored files in the input directory
     -o  | --output-directory
     -rl | --resource-list-directory directory with files containing docker, pypi and npm lists
    "
    exit 1
}

publish_ports () {
    for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_PORT}; do
        if [[ ${REGISTRY} != *":"* ]]; then
            if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then
                PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}"
            fi
        else
            REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})"
            if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then
                PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"
            fi
        fi
    done
}

simulated_hosts () {
    SIMUL_HOSTS=($(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true ) ${NEXUS_DOMAIN})
    for HOST in "${SIMUL_HOSTS[@]}"; do
        if ! grep -wq ${HOST} /etc/hosts; then
            echo "127.0.0.1 ${HOST}" >> /etc/hosts
        fi
    done
}

load_docker_images () {
    for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
        docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
    done
}

push_npm () {
    for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${1}); do
        npm publish --access public ${ARCHIVE} > /dev/null
        echo "NPM ${ARCHIVE} pushed to Nexus"
    done
}

push_pip () {
    for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do
        twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}*
        echo "PYPI ${PACKAGE} pushed to Nexus"
done
}

docker_login () {
    for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do
        echo "Docker login to ${REGISTRY}"
        docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" ${REGISTRY} > /dev/null
done
}

push_docker () {
    for IMAGE in $(sed $'s/\r// ; /^#/d' ${1} | awk '{ print $1 }'); do
        PUSH=""
        if [[ ${IMAGE} != *"/"* ]]; then
            PUSH="${DOCKER_REGISTRY}/library/${IMAGE}"
        elif [[ ${IMAGE} == *"${DEFAULT_REGISTRY}"* ]]; then
            if [[ ${IMAGE} == *"/"*"/"* ]]; then
                PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
            else
                PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'\/library/' <<< ${IMAGE})"
            fi
        elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then
            PUSH="${DOCKER_REGISTRY}/${IMAGE}"
        fi
        if [[ ! -z ${PUSH} ]]; then
            docker tag ${IMAGE} ${PUSH}
        else
            PUSH="${IMAGE}"
        fi
            docker push ${PUSH}
        echo "${IMAGE} pushed as ${PUSH} to Nexus"
    done
}

# Verify all dependencies are available in PATH
FAILED_COMMANDS=()
for cmd in ${COMMANDS[*]};
do
    command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
done
if [ ${#FAILED_COMMANDS[*]} -gt 0 ];
then
    echo "Following commands where not found in PATH and are required:"
    echo ${FAILED_COMMANDS[*]}
    echo "Aborting."
    exit 1
fi

while [ "${1}" != "" ]; do
    case ${1} in
        -i | --input-directory )           shift
                                           DATA_DIR="${1}"
                                           ;;
        -ld | --load-docker-images )       DOCKER_LOAD="true"
                                           ;;
        -o | --output-directory )          shift
                                           NEXUS_DATA_DIR="${1}"
                                           ;;
        -rl | --resource-list-directory )  shift
                                           LISTS_DIR="${1}"
                                           ;;
        -h | --help )                      usage
                                           ;;
        *)                                 usage
    esac
    shift
done

# Setup directories with resources for docker, npm and pypi
NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"

# Setup specific resources lists
NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list"
NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list"

# Setup Nexus image used for build and install infra
INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
NEXUS_IMAGE="$(grep sonatype/nexus3 ${INFRA_LIST})"
NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"

# Backup /etc/hosts
HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)"
cp /etc/hosts /etc/${HOSTS_BACKUP}

# Backup the current docker registry settings
if [ -f ~/.docker/config.json ]; then
    DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)"
    mv ~/.docker/config.json ~/.docker/${DOCKER_CONF_BACKUP}
fi

# Setup default ports published to host as docker registry
PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}"

# Setup additional ports published to host based on simulated docker registries
publish_ports "${NXS_DOCKER_IMG_LIST}"

# Setup simulated domain names to be able to push all to private Nexus repository
simulated_hosts "${NXS_DOCKER_IMG_LIST}"

# Nexus repository configuration setup
NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager
import org.sonatype.nexus.repository.attributes.AttributesFacet
import org.sonatype.nexus.security.user.UserManager
import org.sonatype.nexus.repository.manager.RepositoryManager
import org.sonatype.nexus.security.user.UserNotFoundException
/* Use the container to look up some services. */
realmManager = container.lookup(RealmManager.class)
userManager = container.lookup(UserManager.class, "default") //default user manager
repositoryManager = container.lookup(RepositoryManager.class)
/* Managers are used when scripting api cannot. Note that scripting api can only create mostly, and that creation methods return objects of created entities. */
/* Perform cleanup by removing all repos and users. Realms do not need to be re-disabled, admin and anonymous user will not be removed. */
userManager.listUserIds().each({ id ->
    if (id != "anonymous" && id != "admin")
        userManager.deleteUser(id)
})
repositoryManager.browse().each {
    repositoryManager.delete(it.getName())
}
/* Add bearer token realms at the end of realm lists... */
realmManager.enableRealm("NpmToken")
realmManager.enableRealm("DockerToken")
realmManager.enableRealm("PypiToken")
/* Create the docker user. */
security.addUser("docker", "docker", "docker", "docker@example.com", true, "docker", ["nx-anonymous"])
/* Create docker, npm and pypi repositories. Their default configuration should be compliant with our requirements, except the docker registry creation. */
repository.createNpmHosted("npm-private")
repository.createPyPiHosted("pypi-private")
def r = repository.createDockerHosted("onap", 8082, 0)
/* force basic authentication true by default, must set to false for docker repo. */
conf=r.getConfiguration()
conf.attributes("docker").set("forceBasicAuth", false)
repositoryManager.update(conf)'

# Prepare the Nexus configuration
NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc  '{"name":"configure", "type":"groovy", "content":.}')

#################################
# Docker repository preparation #
#################################

if [ "${DOCKER_LOAD}" == "true" ]; then
    # Load predefined Nexus image
    docker load -i ${NEXUS_IMAGE_TAR}
    # Load all necessary images
    load_docker_images ${NXS_DOCKER_IMG_LIST}
fi

################################
# Nexus repository preparation #
################################

# Prepare nexus-data directory
if [ -d ${NEXUS_DATA_DIR} ]; then
   if [ "$(docker ps -q -f name="${NEXUS_DOMAIN}")" ]; then
       echo "Removing container ${NEXUS_DOMAIN}"
       docker rm -f $(docker ps -aq -f name="${NEXUS_DOMAIN}")
   fi
   pushd ${NEXUS_DATA_DIR}/..
   NXS_BACKUP="$(eval ${TIMESTAMP})_$(basename ${NEXUS_DATA_DIR})_bk"
   mv ${NEXUS_DATA_DIR} "${NXS_BACKUP}"
   echo "${NEXUS_DATA_DIR} already exists - backing up to ${NXS_BACKUP}"
   popd
fi

mkdir -p ${NEXUS_DATA_DIR}
chown 200:200 ${NEXUS_DATA_DIR}
chmod 777 ${NEXUS_DATA_DIR}

# Save Nexus version to prevent/catch data incompatibility
docker images --no-trunc | grep sonatype/nexus3 | awk '{ print $1":"$2" "$3}' > ${NEXUS_DATA_DIR}/nexus.ver

# Start the Nexus
NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE})
echo "Waiting for Nexus to fully start"
until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
    printf "."
    sleep 3
done
echo -e "\nNexus started"

# Configure the nexus repository
curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script
curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null

###########################
# Populate NPM repository #
###########################

# Configure NPM registry to our Nexus repository
echo "Configure NPM registry to ${NPM_REGISTRY}"
npm config set registry "${NPM_REGISTRY}"

# Login to NPM registry
/usr/bin/expect <<EOF
spawn npm login
expect "Username:"
send "${NEXUS_USERNAME}\n"
expect "Password:"
send "${NEXUS_PASSWORD}\n"
expect Email:
send "${NEXUS_EMAIL}\n"
expect eof
EOF

# Patch problematic package
pushd ${NXS_SRC_NPM_DIR}
PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then
    tar xzf "${PATCHED_NPM}"
    rm -f "${PATCHED_NPM}"
    sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
    tar -zcf "${PATCHED_NPM}" package
    rm -rf package
fi

# Push NPM packages to Nexus repository
push_npm "${NXS_NPM_LIST}"
popd

###############################
##  Populate PyPi repository  #
###############################

pushd ${NXS_SRC_PYPI_DIR}
push_pip "${NXS_PYPI_LIST}"
popd

###############################
## Populate Docker repository #
###############################

# Login to simulated docker registries
# Push images to private nexus based on the list
# Images from default registry need to be tagged to private registry
# and those without defined repository in tag uses default repository 'library'
docker_login "${NXS_DOCKER_IMG_LIST}"
push_docker "${NXS_DOCKER_IMG_LIST}"

##############################
# Stop the Nexus and cleanup #
##############################

echo "Stopping Nexus and returning backups"

# Stop the Nexus
docker stop ${NEXUS_CONT_ID} > /dev/null

# Return backed up configuration files
mv -f "/etc/${HOSTS_BACKUP}" /etc/hosts

if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
    mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
fi

# Return default settings
npm config set registry "https://registry.npmjs.org"

echo "Nexus blob is built"
exit 0