| #! /usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| |
| # COPYRIGHT NOTICE STARTS HERE |
| |
| # Copyright 2019 © Samsung Electronics Co., Ltd. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # COPYRIGHT NOTICE ENDS HERE |
| |
| import argparse |
| import datetime |
| import logging |
| import sys |
| import timeit |
| |
| import docker_downloader |
| import git_downloader |
| import http_downloader |
| import npm_downloader |
| import pypi_downloader |
| import rpm_downloader |
| |
| log = logging.getLogger(name=__name__) |
| |
| |
| def parse_args(): |
| """ |
| Parse command line arguments |
| :return: arguments |
| """ |
| parser = argparse.ArgumentParser(description='Download data from lists') |
| list_group = parser.add_argument_group() |
| list_group.add_argument('--docker', action='append', nargs='+', default=[], |
| metavar=('list', 'dir-name'), |
| help='Docker type list. If second argument is specified ' |
| 'it is treated as directory where images will be saved ' |
| 'otherwise only pull operation is executed this can\'t ' |
| 'be mixed between multiple docker list specifications. ' |
| 'if one of the list does not have directory specified ' |
| 'all lists are only pulled!!!') |
| list_group.add_argument('--http', action='append', nargs=2, default=[], |
| metavar=('list', 'dir-name'), |
| help='Http type list and directory to save downloaded files') |
| list_group.add_argument('--npm', action='append', nargs=2, default=[], |
| metavar=('list', 'dir-name'), |
| help='npm type list and directory to save downloaded files') |
| list_group.add_argument('--rpm', action='append', nargs=2, default=[], |
| metavar=('list', 'dir-name'), |
| help='rpm type list and directory to save downloaded files') |
| list_group.add_argument('--git', action='append', nargs=2, default=[], |
| metavar=('list', 'dir-name'), |
| help='git repo type list and directory to save downloaded files') |
| list_group.add_argument('--pypi', action='append', nargs=2, default=[], |
| metavar=('list', 'dir-name'), |
| help='pypi packages type list and directory to save downloaded files') |
| parser.add_argument('--npm-registry', default='https://registry.npmjs.org', |
| help='npm registry to use (default: https://registry.npmjs.org)') |
| parser.add_argument('--check', '-c', action='store_true', default=False, |
| help='Check what is missing. No download.') |
| parser.add_argument('--debug', action='store_true', default=False, |
| help='Turn on debug output') |
| |
| args = parser.parse_args() |
| |
| for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'): |
| if getattr(args, arg): |
| return args |
| |
| parser.error('One of --docker, --npm, --http, --rpm, --git or --pypi must be specified') |
| |
| |
| def log_start(item_type): |
| """ |
| Log starting message |
| :param item_type: type of resources |
| :return: |
| """ |
| log.info('Starting download of {}.'.format(item_type)) |
| |
| |
| def handle_download(downloader, check_mode, errorred_lists, start_time): |
| """ |
| Handle download of resources |
| :param downloader: downloader to use |
| :param check_mode: run in check mode (boolean) |
| :param errorred_lists: list of data types of failed lists |
| :param start_time: timeit.default_timer() right before download |
| :return: timeit.default_timer() at the end of download |
| """ |
| if check_mode: |
| print(downloader.check_table) |
| else: |
| log_start(downloader.list_type) |
| try: |
| downloader.download() |
| except RuntimeError: |
| errorred_lists.append(downloader.list_type) |
| return log_time_interval(start_time, downloader.list_type) |
| |
| |
| def handle_command_download(downloader_class, check_mode, errorred_lists, start_time, *args): |
| """ |
| Handle download of resources where shell command is used |
| :param downloader_class: Class of command_downloader.CommandDownloader to use |
| :param check_mode: run in check mode (boolean) |
| :param errorred_lists: list of data types of failed lists |
| :param start_time: timeit.default_timer() right before download |
| :param args: arguments for downloader class initialization |
| :return: timeit.default_timer() at the end of download |
| """ |
| try: |
| downloader = downloader_class(*args) |
| return handle_download(downloader, check_mode, errorred_lists, start_time) |
| except FileNotFoundError as err: |
| classname = type(downloader_class).__name__ |
| log.exception('Error initializing: {}: {}'.format(classname, err)) |
| return timeit.default_timer() |
| |
| |
| def log_time_interval(start, resource_type=''): |
| """ |
| Log how long the download took |
| :param start: timeit.default_timer() when interval started |
| :param resource_type: type of data that was downloaded. (empty string for whole download) |
| :return: timeit.default_timer() after logging |
| """ |
| e_time = datetime.timedelta(seconds=timeit.default_timer() - start) |
| if resource_type: |
| msg = 'Download of {} took {}\n'.format(resource_type, e_time) |
| else: |
| msg = 'Execution ended. Total elapsed time {}'.format(e_time) |
| log.info(msg) |
| return timeit.default_timer() |
| |
| |
| def run_cli(): |
| if sys.version_info.major < 3: |
| log.error('Unfortunately Python 2 is not supported for data download.') |
| sys.exit(1) |
| args = parse_args() |
| |
| console_handler = logging.StreamHandler(sys.stdout) |
| console_formatter = logging.Formatter('%(message)s') |
| console_handler.setFormatter(console_formatter) |
| now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') |
| log_file = 'download_data-{}.log'.format(now) |
| file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s" |
| |
| if args.debug: |
| logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format) |
| else: |
| logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format) |
| root_logger = logging.getLogger() |
| root_logger.addHandler(console_handler) |
| |
| errorred_lists = [] |
| timer_start = interval_start = timeit.default_timer() |
| |
| if args.check: |
| log.info('Check mode. No download will be executed.') |
| |
| if args.docker: |
| save = True if len(list(filter(lambda x: len(x) == 2, args.docker))) == len(args.docker) else False |
| docker = docker_downloader.DockerDownloader(save, *args.docker, workers=3) |
| interval_start = handle_download(docker, args.check, errorred_lists, interval_start) |
| |
| if args.http: |
| http = http_downloader.HttpDownloader(*args.http) |
| interval_start = handle_download(http, args.check, errorred_lists, interval_start) |
| |
| if args.npm: |
| npm = npm_downloader.NpmDownloader(args.npm_registry, *args.npm) |
| interval_start = handle_download(npm, args.check, errorred_lists, interval_start) |
| |
| if args.rpm: |
| interval_start = handle_command_download(rpm_downloader.RpmDownloader, args.check, errorred_lists, |
| interval_start, *args.rpm) |
| |
| if args.git: |
| interval_start = handle_command_download(git_downloader.GitDownloader, args.check, errorred_lists, |
| interval_start, *args.git) |
| |
| if args.pypi: |
| handle_command_download(pypi_downloader.PyPiDownloader, args.check, errorred_lists, |
| interval_start, *args.pypi) |
| |
| if not args.check: |
| log_time_interval(timer_start) |
| |
| if errorred_lists: |
| log.error('Errors encountered while processing these types:' |
| '\n{}'.format('\n'.join(errorred_lists))) |
| sys.exit(1) |
| |
| |
| if __name__ == '__main__': |
| run_cli() |