build/download/http_downloader.py

#! /usr/bin/env python
# -*- coding: utf-8 -*-

#   COPYRIGHT NOTICE STARTS HERE

#   Copyright 2019 © Samsung Electronics Co., Ltd.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

#   COPYRIGHT NOTICE ENDS HERE

import argparse
import datetime
import logging
import os
import sys
import timeit

import requests
from retrying import retry

import http_file
from concurrent_downloader import ConcurrentDownloader

log = logging.getLogger(__name__)


class HttpDownloader(ConcurrentDownloader):
    def __init__(self, *list_args, list_type='http_files', workers=None):
        super().__init__(list_type, *list_args, workers=workers)

    @property
    def check_table(self):
        """
        Table with information what items from lists are downloaded
        """
        self.missing()
        header = ['Name', 'Downloaded']
        return self._check_table(header, {'Name': 'l'},
                                 ((item, item not in self._missing) for item
                                  in self._data_list))

    @staticmethod
    def _make_get_request(url):
        """
        Run http get request
        :param url: url to reqeuest
        :return: requests.Response
        """
        req = requests.get(url)
        req.raise_for_status()
        return req

    def _is_missing(self, item):
        """
        Check if item is missing (not downloaded)
        :param item: item to check
        :return: boolean
        """
        return not os.path.isfile(
            '{}/{}'.format(self._data_list[item], item.rsplit('//')[-1]))

    @retry(stop_max_attempt_number=5, wait_fixed=2000)
    def _get_file(self, file_uri):
        """
        Get http file from uri
        :param file_uri: uri of the file
        :return: file content
        """
        if not file_uri.startswith('http'):
            file_uri = 'http://' + file_uri
        file_req = self._make_get_request(file_uri)
        return file_req.content

    def _download_item(self, item):
        """
        Download http file
        :param item: http file to be downloaded (tuple: (uri, dst_dir))
        """
        log.info('Downloading: {}'.format(item[0]))
        dst_path = '{}/{}'.format(item[1], item[0].rsplit('//')[-1])
        try:
            f = http_file.HttpFile(item[0], self._get_file(item[0]), dst_path)
            f.save_to_file()
        except Exception as err:
            log.exception('Error downloading: {}: {}'.format(item[0], err))
            if os.path.isfile(dst_path):
                os.remove(dst_path)
            raise err
        log.info('Downloaded: {}'.format(f.name))


def run_cli():
    """
    Run as cli tool
    """
    parser = argparse.ArgumentParser(description='Download http files from list')
    parser.add_argument('file_list', metavar='file-list',
                        help='File with list of http files to download')
    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
                        help='Destination directory for saving')
    parser.add_argument('--check', '-c', action='store_true', default=False,
                        help='Check mode')
    parser.add_argument('--debug', action='store_true', default=False,
                        help='Turn on debug output')
    parser.add_argument('--workers', type=int, default=None,
                        help='Set maximum workers for parallel download (default: cores * 5)')

    args = parser.parse_args()

    if args.debug:
        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
    else:
        logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')

    downloader = HttpDownloader([args.file_list, args.output_dir], workers=args.workers)

    if args.check:
        log.info('Check mode. No download will be executed.')
        log.info(downloader.check_table)
        sys.exit(0)

    timer_start = timeit.default_timer()
    try:
        downloader.download()
    except RuntimeError:
        sys.exit(1)
    finally:
        log.info('Downloading finished in {}'.format(
            datetime.timedelta(seconds=timeit.default_timer() - timer_start)))


if __name__ == '__main__':
    run_cli()