get.py

#!/usr/bin/env python

"""Script to download and extract tools

This script will download and extract required tools into the current directory.
Tools list is obtained from package/package_esp8266com_index.template.json file.
"""

from __future__ import print_function

__author__ = "Ivan Grokhotkov"
__version__ = "2015"

import os
import shutil
import errno
import os.path
import hashlib
import json
import platform
import sys
import tarfile
import zipfile
import re

if sys.version_info[0] == 3:
    from urllib.request import urlretrieve
    from urllib.request import urlopen
    unicode = lambda s: str(s)
else:
    # Not Python 3 - today, it is most likely to be Python 2
    from urllib import urlretrieve
    from urllib import urlopen

if 'Windows' in platform.system():
    import requests

# determine if application is a script file or frozen exe
if getattr(sys, 'frozen', False):
    current_dir = os.path.dirname(os.path.realpath(unicode(sys.executable)))
elif __file__:
    current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))

#current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))
dist_dir = current_dir + '/dist/'

def sha256sum(filename, blocksize=65536):
    hash = hashlib.sha256()
    with open(filename, "rb") as f:
        for block in iter(lambda: f.read(blocksize), b""):
            hash.update(block)
    return hash.hexdigest()

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno != errno.EEXIST or not os.path.isdir(path):
            raise

def report_progress(count, blockSize, totalSize):
    if sys.stdout.isatty():
        if totalSize > 0:
            percent = int(count*blockSize*100/totalSize)
            percent = min(100, percent)
            sys.stdout.write("\r%d%%" % percent)
        else:
            sofar = (count*blockSize) / 1024
            if sofar >= 1000:
                sofar /= 1024
                sys.stdout.write("\r%dMB  " % (sofar))
            else:
                sys.stdout.write("\r%dKB" % (sofar))
        sys.stdout.flush()

def unpack(filename, destination):
    dirname = ''
    print('Extracting {0} ...'.format(os.path.basename(filename)))
    sys.stdout.flush()
    if filename.endswith('tar.gz'):
        tfile = tarfile.open(filename, 'r:gz')
        tfile.extractall(destination)
        dirname = tfile.getnames()[0]
    elif filename.endswith('tar.xz'):
        tfile = tarfile.open(filename, 'r:xz')
        tfile.extractall(destination)
        dirname = tfile.getnames()[0]
    elif filename.endswith('zip'):
        zfile = zipfile.ZipFile(filename)
        zfile.extractall(destination)
        dirname = zfile.namelist()[0]
    else:
        raise NotImplementedError('Unsupported archive type')

    # a little trick to rename tool directories so they don't contain version number
    rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-')
    if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'):
        rename_to = 'esp32-arduino-libs'
    if rename_to != dirname:
        print('Renaming {0} to {1} ...'.format(dirname, rename_to))
        if os.path.isdir(rename_to):
            shutil.rmtree(rename_to)
        shutil.move(dirname, rename_to)

def download_file_with_progress(url,filename):
    import ssl
    import contextlib
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    with contextlib.closing(urlopen(url,context=ctx)) as fp:
        total_size = int(fp.getheader("Content-Length",fp.getheader("Content-length","0")))
        block_count = 0
        block_size = 1024 * 8
        block = fp.read(block_size)
        if block:
            with open(filename,'wb') as out_file:
                out_file.write(block)
                block_count += 1
                report_progress(block_count, block_size, total_size)
                while True:
                    block = fp.read(block_size)
                    if not block:
                        break
                    out_file.write(block)
                    block_count += 1
                    report_progress(block_count, block_size, total_size)
        else:
            raise Exception ('nonexisting file or connection error')

def download_file(url,filename):
    import ssl
    import contextlib
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    with contextlib.closing(urlopen(url,context=ctx)) as fp:
        block_size = 1024 * 8
        block = fp.read(block_size)
        if block:
            with open(filename,'wb') as out_file:
                out_file.write(block)
                while True:
                    block = fp.read(block_size)
                    if not block:
                        break
                    out_file.write(block)
        else:
            raise Exception ('nonexisting file or connection error')

def get_tool(tool):
    sys_name = platform.system()
    archive_name = tool['archiveFileName']
    local_path = dist_dir + archive_name
    url = tool['url']
    if not os.path.isfile(local_path):
        print('Downloading ' + archive_name + ' ...')
        sys.stdout.flush()
        if 'CYGWIN_NT' in sys_name:
            import ssl
            ctx = ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode = ssl.CERT_NONE
            urlretrieve(url, local_path, report_progress, context=ctx)
        elif 'Windows' in sys_name:
            r = requests.get(url)
            f = open(local_path, 'wb')
            f.write(r.content)
            f.close()
        else:
            is_ci = os.environ.get('GITHUB_WORKSPACE');
            if is_ci:
                download_file(url, local_path)
            else:
                try:
                    urlretrieve(url, local_path, report_progress)
                except:
                    download_file_with_progress(url, local_path)
                sys.stdout.write("\rDone   \n")
                sys.stdout.flush()
    else:
        print('Tool {0} already downloaded'.format(archive_name))
        sys.stdout.flush()
    unpack(local_path, '.')

def load_tools_list(filename, platform):
    tools_info = json.load(open(filename))['packages'][0]['tools']
    tools_to_download = []
    for t in tools_info:
        tool_platform = [p for p in t['systems'] if p['host'] == platform]
        if len(tool_platform) == 0:
            # Fallback to x86 on Apple ARM
            if platform == 'arm64-apple-darwin':
                tool_platform = [p for p in t['systems'] if p['host'] == 'x86_64-apple-darwin']
                if len(tool_platform) == 0:
                    continue
            # Fallback to 32bit on 64bit x86 Windows
            elif platform == 'x86_64-mingw32':
                tool_platform = [p for p in t['systems'] if p['host'] == 'i686-mingw32']
                if len(tool_platform) == 0:
                    continue
            else:
                continue
        tools_to_download.append(tool_platform[0])
    return tools_to_download

def identify_platform():
    arduino_platform_names = {'Darwin'   : {32 : 'i386-apple-darwin',   64 : 'x86_64-apple-darwin'},
                              'DarwinARM': {32 : 'arm64-apple-darwin',  64 : 'arm64-apple-darwin'},
                              'Linux'    : {32 : 'i686-pc-linux-gnu',   64 : 'x86_64-pc-linux-gnu'},
                              'LinuxARM' : {32 : 'arm-linux-gnueabihf', 64 : 'aarch64-linux-gnu'},
                              'Windows'  : {32 : 'i686-mingw32',        64 : 'x86_64-mingw32'}}
    bits = 32
    if sys.maxsize > 2**32:
        bits = 64
    sys_name = platform.system()
    sys_platform = platform.platform()
    if 'Darwin' in sys_name and (sys_platform.find('arm') > 0 or sys_platform.find('arm64') > 0):
        sys_name = 'DarwinARM'
    if 'Linux' in sys_name and (sys_platform.find('arm') > 0 or sys_platform.find('aarch64') > 0):
        sys_name = 'LinuxARM'
    if 'CYGWIN_NT' in sys_name:
        sys_name = 'Windows'
    print('System: %s, Bits: %d, Info: %s' % (sys_name, bits, sys_platform))
    return arduino_platform_names[sys_name][bits]

if __name__ == '__main__':
    is_test = (len(sys.argv) > 1 and sys.argv[1] == '-h')
    if is_test:
        print('Test run!')
    identified_platform = identify_platform()
    print('Platform: {0}'.format(identified_platform))
    tools_to_download = load_tools_list(current_dir + '/../package/package_esp32_index.template.json', identified_platform)
    mkdir_p(dist_dir)
    for tool in tools_to_download:
        if is_test:
            print('Would install: {0}'.format(tool['archiveFileName']))
        else:
            get_tool(tool)
    print('Platform Tools Installed')