Fix download scripts to handle empty lines
Fixing download and saving scripts to be able to ignore empty or 00D
ending lines in lists
Issue-ID: OOM-1818
Change-Id: I55ef1dfbea628f8c1d4b19745536e629a2dbe0c9
Signed-off-by: Tomáš Levora <t.levora@partner.samsung.com>
diff --git a/build/common-functions.sh b/build/common-functions.sh
index e39c477..04ea201 100755
--- a/build/common-functions.sh
+++ b/build/common-functions.sh
@@ -90,3 +90,9 @@
done
}
+clean_list() {
+ sed -e 's/\s*#.*$//' \
+ -e '/^\s*$/d' ${1} |
+ tr -d '\r' |
+ awk '{ print $1 }'
+}
diff --git a/build/creating_data/download-docker-images.sh b/build/creating_data/download-docker-images.sh
index 121cd5a..c0a0bed 100755
--- a/build/creating_data/download-docker-images.sh
+++ b/build/creating_data/download-docker-images.sh
@@ -19,35 +19,21 @@
# COPYRIGHT NOTICE ENDS HERE
-# boilerplate
-RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh'
-if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then
- SCRIPT_DIR=$(dirname "${0}")
- LOCAL_PATH=$(readlink -f "$SCRIPT_DIR")
- . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh
-fi
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
-SRC_IMAGE_LIST=$1
-if [[ -z "$SRC_IMAGE_LIST" ]]; then
- SRC_IMAGE_LIST="docker_image_list.txt"
+LIST_FILE="${1}"
+if [[ -z "$LIST_FILE" ]]; then
+ LIST_FILE="docker_image_list.txt"
fi
echo "Download all images"
-lines=$(cat $SRC_IMAGE_LIST | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
line=1
-while read -r image; do
+for image in $(clean_list "$LIST_FILE"); do
echo "== pkg #$line of $lines =="
-
- name=$(echo $image|awk '{print $1}')
- digest=$(echo $image|awk '{print $2}')
-
- echo "$name digest:$digest"
- if [[ "$digest" == "<none>" ]]; then
- retry docker -l error pull "$name"
- else
- retry docker -l error pull "$name"
- fi
+ echo "$image"
+ retry docker -l error pull "$image"
line=$((line+1))
-
-done < "$SRC_IMAGE_LIST"
+done
diff --git a/build/creating_data/download-files.sh b/build/creating_data/download-files.sh
index 89e2026..f687fda 100755
--- a/build/creating_data/download-files.sh
+++ b/build/creating_data/download-files.sh
@@ -16,7 +16,11 @@
#
# COPYRIGHT NOTICE ENDS HERE
-LIST_FILE="$1"
+
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
+LIST_FILE="${1}"
if [[ -z "$LIST_FILE" ]]; then
echo "Missing list file"
exit 1
@@ -28,13 +32,13 @@
exit 1
fi
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
cnt=1
# create output dir if not exists
mkdir -p "$outdir"
-while read -r line; do
+for line in $(clean_list "$LIST_FILE"); do
# www.springframework.org/schema/tool/spring-tool-4.3.xsd
file="${line%%\?*}"
filename=$(basename "$file")
@@ -43,5 +47,4 @@
# drop below 10b/10s
curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$filename" &>/dev/null
cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
\ No newline at end of file
diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh
index 06f4135..1144c66 100755
--- a/build/creating_data/download-http-files.sh
+++ b/build/creating_data/download-http-files.sh
@@ -16,7 +16,11 @@
#
# COPYRIGHT NOTICE ENDS HERE
-LIST_FILE="$1"
+
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
+LIST_FILE="${1}"
if [[ -z "$LIST_FILE" ]]; then
echo "Missing list file"
exit 1
@@ -28,13 +32,13 @@
exit 1
fi
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
cnt=1
# create output dir if not exists
mkdir -p "$outdir"
-while read -r line; do
+for line in $(clean_list "$LIST_FILE"); do
# www.springframework.org/schema/tool/spring-tool-4.3.xsd
file="${line%%\?*}"
echo "Downloading $cnt / $lines: $file"
@@ -44,5 +48,4 @@
# drop below 10b/10s
curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null
cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
\ No newline at end of file
diff --git a/build/creating_data/download-npm-pkgs.sh b/build/creating_data/download-npm-pkgs.sh
index 9e8847c..191dd5d 100755
--- a/build/creating_data/download-npm-pkgs.sh
+++ b/build/creating_data/download-npm-pkgs.sh
@@ -16,7 +16,10 @@
#
# COPYRIGHT NOTICE ENDS HERE
-LIST_FILE="$1"
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
+LIST_FILE="${1}"
if [[ -z "$LIST_FILE" ]]; then
LIST_FILE="all_npm_list.txt"
@@ -30,12 +33,10 @@
mkdir -p "$outdir"
cd "$outdir"
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
cnt=1
-while read -r line; do
+for line in $(clean_list "$LIST_FILE"); do
echo "== pkg #$cnt of $lines =="
- # yallist@2.1.2
npm pack $line
cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
\ No newline at end of file
diff --git a/build/creating_data/download-pip.sh b/build/creating_data/download-pip.sh
index dea60b5..fd4a436 100755
--- a/build/creating_data/download-pip.sh
+++ b/build/creating_data/download-pip.sh
@@ -16,6 +16,10 @@
#
# COPYRIGHT NOTICE ENDS HERE
+
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
LIST_FILE="$1"
if [[ -z "$LIST_FILE" ]]; then
echo "Missing list file"
@@ -30,16 +34,15 @@
exit 1
fi
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
cnt=1
# create output dir if not exists
mkdir -p "$outdir"
cd "$outdir"
-while read -r line; do
+for line in $(clean_list "$LIST_FILE)"; do
echo "Downloading $cnt / $lines: $line"
pip download $line
cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
diff --git a/build/creating_data/save-docker-images.sh b/build/creating_data/save-docker-images.sh
index 4c76455..0a72d15 100755
--- a/build/creating_data/save-docker-images.sh
+++ b/build/creating_data/save-docker-images.sh
@@ -19,15 +19,10 @@
# COPYRIGHT NOTICE ENDS HERE
-# boilerplate
-RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh'
-if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then
- SCRIPT_DIR=$(dirname "${0}")
- LOCAL_PATH=$(readlink -f "$SCRIPT_DIR")
- . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh
-fi
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
-LIST="${1}"
+LIST_FILE="${1}"
IMG_DIR="${2}"
if [[ -z "$IMG_DIR" ]]; then
@@ -56,11 +51,9 @@
echo "Save all images"
line=1
-lines=$(wc -l ${LIST})
-while read -r image; do
+lines=$(clean_list "$LIST_FILE" | wc -l)
+for image in $(clean_list "$LIST_FILE"); do
echo "== pkg #$line of $lines =="
-
save_image "${image}"
line=$((line+1))
-
-done < "${LIST}"
+done
\ No newline at end of file