From: Cian Johnston Date: Thu, 4 Feb 2021 14:51:58 +0000 (+0000) Subject: containers: export all images in one file X-Git-Url: https://gerrit.nordix.org/gitweb?p=infra%2Fstack%2Fkubernetes.git;a=commitdiff_plain;h=dbb881e7fd077d2818d01dd6b46d4899af6940d8 containers: export all images in one file This change modifies how images are exported before packaging. Previously, each image was exported as a separate file, which causes duplication of layers in common between images. Now, all images are exported as one file, allowing deduplication of image layers and significant on-disk space savings. NOTE: the docker-registry image is exported separately for the sake of engine. Change-Id: I95a0597590ada9c32c8b9449d6f58b321168c2f8 Signed-off-by: Cian Johnston --- diff --git a/playbooks/roles/package/files/pull-images.sh b/playbooks/roles/package/files/pull-images.sh new file mode 100755 index 0000000..3ceec85 --- /dev/null +++ b/playbooks/roles/package/files/pull-images.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2021 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + + +set -o nounset +set -o errexit +set -o pipefail + +#------------------------------------------------------------------------------- +# Print the help message which includes the usage, the expected parameters +# and their default values if they are not specified +#------------------------------------------------------------------------------- +function usage() { + + # NOTE: shellcheck complains quoting in the example so SC2086 is disabled + # shellcheck disable=SC2086 + cat <] [-p ] [-c ] [-v] [-h] + + -l: File with list of docker images. (Default /tmp/docker-list.tmp) + -c: container command. (Default docker) + -p: number in parallel. (Default 4) + -v: Increase verbosity and keep logs for troubleshooting. (Default false) + -h: This message. + +EOF + exit 0 + +} + + +#------------------------------------------------------------------------------- +# Parse the arguments that are passed to the script +# If an argument is not specified, default values for those are set +# +# The priority order is +# - arguments: overrides the default values and values set as environment +# values. highest prio. +# - env vars: overrides the default values but not the values set from command +# line. +# - default values: only takes effect if the user doesn't specify the value +# of an argument either as an env var or from command line. lowest prio. +#------------------------------------------------------------------------------- +function parse_cmdline_opts() { + + # set variables to the values set in env - otherwise, set them to defaults + LISTOFDOCKERIMAGES=${LISTOFDOCKERIMAGES:-"/tmp/docker-list.tmp"} + PARALLELNU=${PARALLELNU:-4} + VERBOSITY="false" + CONTAINER_TOOL="docker" + + while getopts ":hl:p:c:v" o; do + case "${o}" in + h) usage ;; + l) LISTOFDOCKERIMAGES="${OPTARG}" ;; + p) PARALLELNU="${OPTARG}" ;; + c) CONTAINER_TOOL="${OPTARG}" ;; + v) VERBOSITY="true" ;; + *) echo "ERROR: Invalid option '-${OPTARG}'"; usage ;; + esac + done + + # Do all the exports + export LISTOFDOCKERIMAGES="${LISTOFDOCKERIMAGES}" + export PARALLELNU="${PARALLELNU}" + export VERBOSITY="${VERBOSITY}" + export CONTAINER_TOOL="${CONTAINER_TOOL}" +} + +parse_cmdline_opts "$@" + +xargs -n 1 -P "$PARALLELNU" "$CONTAINER_TOOL" pull < "$LISTOFDOCKERIMAGES" diff --git a/playbooks/roles/package/files/save-images.sh b/playbooks/roles/package/files/save-images.sh new file mode 100755 index 0000000..0b2cb74 --- /dev/null +++ b/playbooks/roles/package/files/save-images.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2021 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +set -o errexit +set -o nounset +set -o pipefail + +#------------------------------------------------------------------------------- +# Print the help message which includes the usage, the expected parameters +# and their default values if they are not specified +#------------------------------------------------------------------------------- +function usage() { + + # NOTE: shellcheck complains quoting in the example so SC2086 is disabled + # shellcheck disable=SC2086 + cat <] [-c ] [-o ] [-v] [-h] + + -l: File with list of docker images. (Default /tmp/dockersave-list.tmp) + -c: container command. (Default docker) + -v: Increase verbosity and keep logs for troubleshooting. (Default false) + -o: Path in which to save the images. (Default /tmp/images.tgz) + -h: This message. + +EOF + exit 0 + +} + +#------------------------------------------------------------------------------- +# Parse the arguments that are passed to the script +# If an argument is not specified, default values for those are set +# +# The priority order is +# - arguments: overrides the default values and values set as environment +# values. highest prio. +# - env vars: overrides the default values but not the values set from command +# line. +# - default values: only takes effect if the user doesn't specify the value +# of an argument either as an env var or from command line. lowest prio. +#------------------------------------------------------------------------------- +function parse_cmdline_opts() { + + # set variables to the values set in env - otherwise, set them to defaults + LISTOFDOCKERIMAGES=${LISTOFDOCKERIMAGES:-"/tmp/dockersave-list.tmp"} + OUTPUT_PATH=${OUTPUT_PATH:-"/tmp/images.tgz"} + VERBOSITY="false" + CONTAINER_TOOL="docker" + + while getopts "hl:c:o:v" o; do + case "${o}" in + h) usage ;; + l) LISTOFDOCKERIMAGES="${OPTARG}" ;; + c) CONTAINER_TOOL="${OPTARG}" ;; + o) OUTPUT_PATH="${OPTARG}" ;; + v) VERBOSITY="true" ;; + *) echo "ERROR: Invalid option '-${OPTARG}'"; usage ;; + esac + done + + # Do all the exports + export LISTOFDOCKERIMAGES="${LISTOFDOCKERIMAGES}" + export VERBOSITY="${VERBOSITY}" + export CONTAINER_TOOL="${CONTAINER_TOOL}" +} + +parse_cmdline_opts "$@" + +mkdir -p "$(dirname "${OUTPUT_PATH}")" +xargs "${CONTAINER_TOOL}" save < "${LISTOFDOCKERIMAGES}" | gzip --rsyncable - > "${OUTPUT_PATH}" diff --git a/playbooks/roles/package/tasks/containers.yaml b/playbooks/roles/package/tasks/containers.yaml index c90f906..e94d6fd 100644 --- a/playbooks/roles/package/tasks/containers.yaml +++ b/playbooks/roles/package/tasks/containers.yaml @@ -83,73 +83,70 @@ kubeadm_images: "{{ kubeadm_images_cooked.results | map(attribute='ansible_facts.kubeadm_image') | list | items2dict }}" run_once: true -# NOTE (fdegir): docker_image module doesn't seem to respect become so falling back to command module -- name: Pull kubeadm container images - command: "docker pull {{ kubeadm_images[item.key].repo }}:{{ kubeadm_images[item.key].tag }}" - with_dict: "{{ kubeadm_images }}" - become: true +- name: Populate list of images to pull and save with those specified by Kubespray + vars: + image_list: [] + set_fact: + image_list: "{{ image_list }} + [ '{{ downloads[item].repo }}:{{ downloads[item].tag }}' ]" + loop: "{{ k8s_misc_images }}" changed_when: false + when: downloads[item].container is defined and downloads[item].container -- name: Pull tiller container image (helm v2 only) - command: "docker pull {{ downloads['tiller'].repo }}:{{ downloads['tiller'].tag }}" - become: true +- name: Populate list of images to pull and save with those specified by Kubeadm + set_fact: + image_list: "{{ image_list }} + [ '{{ item.value.repo }}:{{ item.value.tag }}' ]" + loop: "{{ kubeadm_images | dict2items }}" changed_when: false - when: helm_version is version('v3.0.0', '<') -- name: Pull misc container images - command: "docker pull {{ downloads[item].repo }}:{{ downloads[item].tag }}" - loop: "{{ k8s_misc_images }}" - become: true +- name: Populate list of images to pull and save with other miscellaneous images + set_fact: + image_list: "{{ image_list }} + [ '{{ item.value.repo }}:{{ item.value.tag }}' ]" + loop: "{{ other_images | dict2items }}" changed_when: false - when: downloads[item].container is defined and downloads[item].container -- name: Pull other container images - command: "docker pull {{ other_images[item.key].repo }}:{{ other_images[item.key].tag }}" - with_dict: "{{ other_images }}" - become: true +- name: Add tiller container image to list of images (helm v2 only) + set_fact: + image_list: "{{ image_list }} + [ '{{ downloads['tiller'].repo }}:{{ downloads['tiller'].tag }}' ]" changed_when: false + when: helm_version is version('v3.0.0', '<') -# save container images -- name: Save kubeadm container images - command: |- - docker save {{ kubeadm_images[item.key].repo }}:{{ kubeadm_images[item.key].tag }} - -o {{ kubeadm_images[item.key].repo | replace('/', '_') }}_{{ kubeadm_images[item.key].tag }}.tar - with_dict: "{{ kubeadm_images }}" - args: - chdir: "{{ containers_folder }}" - become: true - changed_when: false +- name: Dump image list to disk + copy: + content: "{{ image_list | join('\n') }}" + dest: "{{ engine_cache }}/images.txt" -- name: Save misc container images - command: |- - docker save {{ downloads[item].repo }}:{{ downloads[item].tag }} - -o {{ downloads[item].repo }} -o {{ downloads[item].repo | replace('/', '_') }}_{{ downloads[item].tag }}.tar - loop: "{{ k8s_misc_images }}" - args: - chdir: "{{ containers_folder }}" +- name: Pull images (using pull-images.sh) + vars: + container_tool: docker + container_pull_parallel: 4 + images_txt: "{{ engine_cache }}/images.txt" + script: pull-images.sh -l {{ images_txt }} -c {{ container_tool }} -p {{ container_pull_parallel }} become: true - changed_when: false - when: downloads[item].container is defined and downloads[item].container + register: pull_images_output_raw + changed_when: pull_images_output_raw.stdout is search("downloaded newer image for", ignorecase=true) -- name: Save tiller container image (helm v2 only) - command: |- - docker save {{ downloads['tiller'].repo }}:{{ downloads['tiller'].tag }} - -o {{ downloads['tiller'].repo }} -o {{ downloads['tiller'].repo | replace('/', '_') }}_{{ downloads['tiller'].tag }}.tar +- name: Save images (using save-images.sh) + vars: + container_tool: docker + images_txt: "{{ engine_cache }}/images.txt" + images_output: "{{ containers_folder }}/images.tar" + script: save-images.sh -l {{ images_txt }} -c {{ container_tool }} -o {{ images_output }} args: - chdir: "{{ containers_folder }}" + creates: "{{ images_output }}" become: true - changed_when: false - when: helm_version is version('v3.0.0', '<') -- name: Save other container images - command: |- - docker save {{ other_images[item.key].repo }}:{{ other_images[item.key].tag }} - -o {{ other_images[item.key].repo | replace('/', '_') }}_{{ other_images[item.key].tag }}.tar - with_dict: "{{ other_images }}" +- name: Save registry image separately + vars: + img: "{{ other_images['docker-registry'].repo }}:{{ other_images['docker-registry'].tag }}" + dest: "{{ img | regex_replace('[/:]', '_') }}.tar" + shell: |- + set -o pipefail && \ + docker save {{ img }} -o {{ dest }} args: + executable: "/bin/bash" chdir: "{{ containers_folder }}" + creates: "{{ containers_folder }}/{{ dest }}" become: true - changed_when: false # NOTE (fdegir): archive fails due to wrong permissions so we fix them - name: Fix container image permissions diff --git a/playbooks/roles/prepare-artifacts/tasks/main.yaml b/playbooks/roles/prepare-artifacts/tasks/main.yaml index f55b044..fab1bdf 100644 --- a/playbooks/roles/prepare-artifacts/tasks/main.yaml +++ b/playbooks/roles/prepare-artifacts/tasks/main.yaml @@ -20,7 +20,7 @@ - name: Get list of k8s container image tarfiles find: path: "{{ engine_workspace }}/offline/containers" - patterns: '*.tar' + patterns: ['*.tar', '*.tgz', '*.tar.gz'] register: container_image # NOTE (fdegir): the user may not be member of docker group so we need root