blob: d8b448396c8e41b64d254a694151689938c65c78 [file] [log] [blame]
Milan Verespej87866322019-04-18 14:37:51 +02001#! /usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# COPYRIGHT NOTICE STARTS HERE
5
6# Copyright 2019 © Samsung Electronics Co., Ltd.
7#
8# Licensed under the Apache License, Version 2.0 (the "License");
9# you may not use this file except in compliance with the License.
10# You may obtain a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS,
16# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17# See the License for the specific language governing permissions and
18# limitations under the License.
19
20# COPYRIGHT NOTICE ENDS HERE
21
22
Milan Verespej87866322019-04-18 14:37:51 +020023import concurrent.futures
Milan Verespej9dfa7642019-05-07 14:50:20 +020024import os
25import progressbar
26import prettytable
27import requests
Milan Verespej83033592019-05-20 09:44:23 +020028from distutils.spawn import find_executable
Milan Verespej87866322019-04-18 14:37:51 +020029
30progressbar.streams.wrap_stdout()
31progressbar.streams.wrap_stderr()
32
33
34def load_list(item_list):
35 """
36 Parse list with items to be downloaded.
37 :param item_list: File with list of items (1 line per item)
38 :return: set of items from file
39 """
40 with open(item_list, 'r') as f:
Milan Verespej455be472019-05-23 14:21:19 +020041 return {item for item in (line.strip() for line in f)
42 if item and not item.startswith('#')}
Milan Verespej87866322019-04-18 14:37:51 +020043
44
45def init_progress(items_name):
46 progress_widgets = ['Downloading {}: '.format(items_name),
47 progressbar.Bar(), ' ',
48 progressbar.Percentage(), ' ',
49 '(', progressbar.SimpleProgress(), ')']
50
51 progress = progressbar.ProgressBar(widgets=progress_widgets,
52 poll_rate=1.0,
53 redirect_stdout=True)
54 return progress
55
56
57def start_progress(progress, target_count, skipping, log):
58 log_skipping(skipping, log)
59 log.info("Initializing download. Takes a while.")
60
61 progress.max_value = target_count
62 progress.start()
63 progress.update(len(skipping))
64
65
66def log_skipping(skipping_iterable, logger):
67 for skipped in skipping_iterable:
68 logger.info('Skipping: {}'.format(skipped))
69
70
71def run_concurrent(workers, progress, fn, iterable, *args):
72 with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
73 futures = [executor.submit(fn, item, *args) for item in iterable]
74 error_count = 0
75 for future in concurrent.futures.as_completed(futures):
76 error = future.exception()
77 if error:
78 error_count += 1
79 progress.update()
80 else:
81 progress.update(progress.value +1)
82 return error_count
83
84
85def finish_progress(progress, error_count, log):
86 progress.finish(dirty=error_count > 0)
87 log.info('Download ended. Elapsed time {}'.format(progress.data()['time_elapsed']))
88
Milan Verespej83033592019-05-20 09:44:23 +020089def check_tool(name):
90 return find_executable(name)
Milan Verespej9dfa7642019-05-07 14:50:20 +020091
92def save_to_file(dst, content):
93 """
94 Save downloaded byte content to file
95 :param dst: path to file to save content to
96 :param content: byte content of file
97 """
98 dst_dir = os.path.dirname(dst)
99 if not os.path.exists(dst_dir):
100 os.makedirs(dst_dir)
101 with open(dst, 'wb') as dst_file:
102 dst_file.write(content)
103
104def make_get_request(url):
105 req = requests.get(url)
106 req.raise_for_status()
107 return req
108
109def simple_check_table(target, missing):
110 table = prettytable.PrettyTable(['Name', 'Downloaded'])
111 table.align['Name'] = 'l'
112 for item in sorted(target):
113 table.add_row([item, item not in missing])
114 return table
115