poky/scripts/install-buildtools

#!/usr/bin/env python3

# Buildtools and buildtools extended installer helper script
#
# Copyright (C) 2017-2020 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
#  NOTE: --with-extended-buildtools is on by default
#
#  Example usage (extended buildtools from milestone):
#    (1) using --url and --filename
#        $ install-buildtools \
#          --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M2/buildtools \
#          --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200122.sh
#    (2) using --base-url, --release, --installer-version and --build-date
#        $ install-buildtools \
#          --base-url http://downloads.yoctoproject.org/releases/yocto \
#          --release yocto-3.1_M2 \
#          --installer-version 3.0+snapshot
#          --build-date 202000122
#
#  Example usage (standard buildtools from release):
#    (3) using --url and --filename
#        $ install-buildtools --without-extended-buildtools \
#          --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
#          --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
#    (4) using --base-url, --release and --installer-version
#        $ install-buildtools --without-extended-buildtools \
#          --base-url http://downloads.yoctoproject.org/releases/yocto \
#          --release yocto-3.0.2 \
#          --installer-version 3.0.2
#

import argparse
import logging
import os
import re
import shutil
import stat
import subprocess
import sys
import tempfile
from urllib.parse import quote

scripts_path = os.path.dirname(os.path.realpath(__file__))
lib_path = scripts_path + '/lib'
sys.path = sys.path + [lib_path]
import scriptutils
import scriptpath

# Figure out where is the bitbake/lib/bb since we need bb.utils.md5_file
bitbakepath = scriptpath.add_bitbake_lib_path()
if not bitbakepath:
    sys.stderr.write("Unable to find bitbake by searching parent directory "
                     "of this script or PATH\n")
    sys.exit(1)

PROGNAME = 'install-buildtools'
logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)

DEFAULT_INSTALL_DIR: str = os.path.join(os.path.split(scripts_path)[0],'buildtools')
DEFAULT_BASE_URL: str = 'http://downloads.yoctoproject.org/releases/yocto'
DEFAULT_RELEASE: str = 'yocto-3.1_M2'
DEFAULT_INSTALLER_VERSION: str = '3.0+snapshot'
DEFAULT_BUILDDATE: str = "20200122"


def main():
    global DEFAULT_INSTALL_DIR
    global DEFAULT_BASE_URL
    global DEFAULT_RELEASE
    global DEFAULT_INSTALLER_VERSION
    global DEFAULT_BUILDDATE
    filename: str = ""
    release: str = ""
    buildtools_url: str = ""
    install_dir: str = ""

    parser = argparse.ArgumentParser(
        description="Buildtools installation helper",
        add_help=False)
    parser.add_argument('-u', '--url',
                        help='URL from where to fetch buildtools SDK installer, not '
                             'including filename (optional)\n'
                             'Requires --filename.',
                        action='store')
    parser.add_argument('-f', '--filename',
                        help='filename for the buildtools SDK installer to be installed '
                             '(optional)\nRequires --url',
                        action='store')
    parser.add_argument('-d', '--directory',
                        default=DEFAULT_INSTALL_DIR,
                        help='directory where buildtools SDK will be installed (optional)',
                        action='store')
    parser.add_argument('-r', '--release',
                        default=DEFAULT_RELEASE,
                        help='Yocto Project release string for SDK which will be '
                             'installed (optional)',
                        action='store')
    parser.add_argument('-V', '--installer-version',
                        default=DEFAULT_INSTALLER_VERSION,
                        help='version string for the SDK to be installed (optional)',
                        action='store')
    parser.add_argument('-b', '--base-url',
                        default=DEFAULT_BASE_URL,
                        help='base URL from which to fetch SDK (optional)', action='store')
    parser.add_argument('-t', '--build-date',
                        default=DEFAULT_BUILDDATE,
                        help='Build date of pre-release SDK (optional)', action='store')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--with-extended-buildtools', action='store_true',
                       dest='with_extended_buildtools',
                       default=True,
                       help='enable extended buildtools tarball (on by default)')
    group.add_argument('--without-extended-buildtools', action='store_false',
                       dest='with_extended_buildtools',
                       help='disable extended buildtools (traditional buildtools tarball)')
    parser.add_argument('-c', '--check', help='enable md5 checksum checking',
                        default=True,
                        action='store_true')
    parser.add_argument('-D', '--debug', help='enable debug output',
                        action='store_true')
    parser.add_argument('-q', '--quiet', help='print only errors',
                        action='store_true')

    parser.add_argument('-h', '--help', action='help',
                        default=argparse.SUPPRESS,
                        help='show this help message and exit')

    args = parser.parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)
    elif args.quiet:
        logger.setLevel(logging.ERROR)

    if args.url and args.filename:
        logger.debug("--url and --filename detected. Ignoring --base-url "
                     "--release --installer-version  arguments.")
        filename = args.filename
        buildtools_url = "%s/%s" % (args.url, filename)
    else:
        if args.base_url:
            base_url = args.base_url
        else:
            base_url = DEFAULT_BASE_URL
        if args.release:
            # check if this is a pre-release "milestone" SDK
            m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
                          args.release)
            logger.debug("milestone regex: %s" % m)
            if m and m.group('milestone'):
                logger.debug("release[distro]: %s" % m.group('distro'))
                logger.debug("release[version]: %s" % m.group('version'))
                logger.debug("release[milestone]: %s" % m.group('milestone'))
                if not args.build_date:
                    logger.error("Milestone installers require --build-date")
                else:
                    if args.with_extended_buildtools:
                        filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
                            args.installer_version, args.build_date)
                    else:
                        filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % (
                            args.installer_version, args.build_date)
                    safe_filename = quote(filename)
                    buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
            # regular release SDK
            else:
                if args.with_extended_buildtools:
                    filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version
                else:
                    filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version
                safe_filename = quote(filename)
                buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)

    tmpsdk_dir = tempfile.mkdtemp()
    try:
        # Fetch installer
        logger.info("Fetching buildtools installer")
        tmpbuildtools = os.path.join(tmpsdk_dir, filename)
        ret = subprocess.call("wget -q -O %s %s" %
                              (tmpbuildtools, buildtools_url), shell=True)
        if ret != 0:
            logger.error("Could not download file from %s" % buildtools_url)
            return ret

        # Verify checksum
        if args.check:
            import bb
            logger.info("Fetching buildtools installer checksum")
            check_url = "{}.md5sum".format(buildtools_url)
            checksum_filename = "%s.md5sum" % filename
            tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
            ret = subprocess.call("wget -q -O %s %s" %
                                  (tmpbuildtools_checksum, check_url), shell=True)
            if ret != 0:
                logger.error("Could not download file from %s" % check_url)
                return ret
            regex = re.compile(r"^(?P<md5sum>[0-9a-f]+)\s\s(?P<path>.*/)(?P<filename>.*)$")
            with open(tmpbuildtools_checksum, 'rb') as f:
                original = f.read()
                m = re.search(regex, original.decode("utf-8"))
                logger.debug("md5sum: %s" % m.group('md5sum'))
                logger.debug("path: %s" % m.group('path'))
                logger.debug("filename: %s" % m.group('filename'))
                if filename != m.group('filename'):
                    logger.error("Filename does not match name in checksum")
                    return 1
                md5sum = m.group('md5sum')
            md5value = bb.utils.md5_file(tmpbuildtools)
            if md5sum == md5value:
                logger.info("Checksum success")
            else:
                logger.error("Checksum %s expected. Actual checksum is %s." %
                             (md5sum, md5value))

        # Make installer executable
        logger.info("Making installer executable")
        st = os.stat(tmpbuildtools)
        os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
        logger.debug(os.stat(tmpbuildtools))
        if args.directory:
            install_dir = args.directory
            ret = subprocess.call("%s -d %s -y" %
                                  (tmpbuildtools, install_dir), shell=True)
        else:
            install_dir = "/opt/poky/%s" % args.installer_version
            ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
        if ret != 0:
            logger.error("Could not run buildtools installer")

        # Test installation
        logger.info("Testing installation")
        tool = ""
        m = re.search("extended", tmpbuildtools)
        logger.debug("extended regex: %s" % m)
        if args.with_extended_buildtools and not m:
            logger.info("Ignoring --with-extended-buildtools as filename "
                        "does not contain 'extended'")
        if args.with_extended_buildtools and m:
            tool = 'gcc'
        else:
            tool = 'tar'
        logger.debug("install_dir: %s" % install_dir)
        proc = subprocess.run(". %s/environment-setup-x86_64-pokysdk-linux && which %s" %
                              (install_dir, tool),
                              shell=True, stdout=subprocess.PIPE)
        which_tool = proc.stdout.decode("utf-8")
        logger.debug("which %s: %s" % (tool, which_tool))
        ret = proc.returncode
        if not which_tool.startswith(install_dir):
            logger.error("Something went wrong: %s not found in %s" %
                         (tool, install_dir))
        if ret != 0:
            logger.error("Something went wrong: installation failed")
        else:
            logger.info("Installation successful. Remember to source the "
                        "environment setup script now and in any new session.")
        return ret

    finally:
        # cleanup tmp directory
        shutil.rmtree(tmpsdk_dir)


if __name__ == '__main__':
    try:
        ret = main()
    except Exception:
        ret = 1
        import traceback

        traceback.print_exc()
    sys.exit(ret)