mirror of
https://git.yoctoproject.org/git/poky
synced 2026-01-04 16:10:04 +00:00
58796dc18f
Our least common denominator supported distro is debian-8 which has python 3.4. The whole point of the install-buildtools script is to make it easier on the user to install buildtools tarball. So it needs to run on Python 3.4. The way we checked if the install was successful in the prior version of the script was not workable in python 3.4. Since the environment-setup-... script is currently just exporting environment variables, use os.environ to do the equivalent from values gleaned via regex from the environment-setup-... file. Corrected a couple minor whitespace errors NOTE: License changed to GPL-2.0-only due to inclusion of code copied directly from bitbake/lib/bb/utils.py. This avoids the need to depend on bitbake, which is now Python 3.5+ only. (From OE-Core rev: 869020dac889e9ed79a294f308a87cfd946a68bd) Signed-off-by: Tim Orling <timothy.t.orling@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
340 lines
14 KiB
Python
Executable File
340 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Buildtools and buildtools extended installer helper script
|
|
#
|
|
# Copyright (C) 2017-2020 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
# NOTE: --with-extended-buildtools is on by default
|
|
#
|
|
# Example usage (extended buildtools from milestone):
|
|
# (1) using --url and --filename
|
|
# $ install-buildtools \
|
|
# --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
|
|
# --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
|
|
# (2) using --base-url, --release, --installer-version and --build-date
|
|
# $ install-buildtools \
|
|
# --base-url http://downloads.yoctoproject.org/releases/yocto \
|
|
# --release yocto-3.1_M3 \
|
|
# --installer-version 3.0+snapshot
|
|
# --build-date 202000315
|
|
#
|
|
# Example usage (standard buildtools from release):
|
|
# (3) using --url and --filename
|
|
# $ install-buildtools --without-extended-buildtools \
|
|
# --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
|
|
# --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
|
|
# (4) using --base-url, --release and --installer-version
|
|
# $ install-buildtools --without-extended-buildtools \
|
|
# --base-url http://downloads.yoctoproject.org/releases/yocto \
|
|
# --release yocto-3.0.2 \
|
|
# --installer-version 3.0.2
|
|
#
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import shlex
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from urllib.parse import quote
|
|
|
|
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
|
lib_path = scripts_path + '/lib'
|
|
sys.path = sys.path + [lib_path]
|
|
import scriptutils
|
|
import scriptpath
|
|
|
|
|
|
PROGNAME = 'install-buildtools'
|
|
logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
|
|
|
|
DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
|
|
DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
|
|
DEFAULT_RELEASE = 'yocto-3.1_M3'
|
|
DEFAULT_INSTALLER_VERSION = '3.0+snapshot'
|
|
DEFAULT_BUILDDATE = "20200315"
|
|
|
|
# Python version sanity check
|
|
if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
|
|
logger.error("This script requires Python 3.4 or greater")
|
|
logger.error("You have Python %s.%s" %
|
|
(sys.version_info.major, sys.version_info.minor))
|
|
sys.exit(1)
|
|
|
|
# The following three functions are copied directly from
|
|
# bitbake/lib/bb/utils.py, in order to allow this script
|
|
# to run on versions of python earlier than what bitbake
|
|
# supports (e.g. less than Python 3.5 for YP 3.1 release)
|
|
|
|
def _hasher(method, filename):
|
|
import mmap
|
|
|
|
with open(filename, "rb") as f:
|
|
try:
|
|
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
|
|
for chunk in iter(lambda: mm.read(8192), b''):
|
|
method.update(chunk)
|
|
except ValueError:
|
|
# You can't mmap() an empty file so silence this exception
|
|
pass
|
|
return method.hexdigest()
|
|
|
|
|
|
def md5_file(filename):
|
|
"""
|
|
Return the hex string representation of the MD5 checksum of filename.
|
|
"""
|
|
import hashlib
|
|
return _hasher(hashlib.md5(), filename)
|
|
|
|
def sha256_file(filename):
|
|
"""
|
|
Return the hex string representation of the 256-bit SHA checksum of
|
|
filename.
|
|
"""
|
|
import hashlib
|
|
return _hasher(hashlib.sha256(), filename)
|
|
|
|
|
|
def main():
|
|
global DEFAULT_INSTALL_DIR
|
|
global DEFAULT_BASE_URL
|
|
global DEFAULT_RELEASE
|
|
global DEFAULT_INSTALLER_VERSION
|
|
global DEFAULT_BUILDDATE
|
|
filename = ""
|
|
release = ""
|
|
buildtools_url = ""
|
|
install_dir = ""
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Buildtools installation helper",
|
|
add_help=False)
|
|
parser.add_argument('-u', '--url',
|
|
help='URL from where to fetch buildtools SDK installer, not '
|
|
'including filename (optional)\n'
|
|
'Requires --filename.',
|
|
action='store')
|
|
parser.add_argument('-f', '--filename',
|
|
help='filename for the buildtools SDK installer to be installed '
|
|
'(optional)\nRequires --url',
|
|
action='store')
|
|
parser.add_argument('-d', '--directory',
|
|
default=DEFAULT_INSTALL_DIR,
|
|
help='directory where buildtools SDK will be installed (optional)',
|
|
action='store')
|
|
parser.add_argument('-r', '--release',
|
|
default=DEFAULT_RELEASE,
|
|
help='Yocto Project release string for SDK which will be '
|
|
'installed (optional)',
|
|
action='store')
|
|
parser.add_argument('-V', '--installer-version',
|
|
default=DEFAULT_INSTALLER_VERSION,
|
|
help='version string for the SDK to be installed (optional)',
|
|
action='store')
|
|
parser.add_argument('-b', '--base-url',
|
|
default=DEFAULT_BASE_URL,
|
|
help='base URL from which to fetch SDK (optional)', action='store')
|
|
parser.add_argument('-t', '--build-date',
|
|
default=DEFAULT_BUILDDATE,
|
|
help='Build date of pre-release SDK (optional)', action='store')
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument('--with-extended-buildtools', action='store_true',
|
|
dest='with_extended_buildtools',
|
|
default=True,
|
|
help='enable extended buildtools tarball (on by default)')
|
|
group.add_argument('--without-extended-buildtools', action='store_false',
|
|
dest='with_extended_buildtools',
|
|
help='disable extended buildtools (traditional buildtools tarball)')
|
|
parser.add_argument('-c', '--check', help='enable md5 checksum checking',
|
|
default=True,
|
|
action='store_true')
|
|
parser.add_argument('-D', '--debug', help='enable debug output',
|
|
action='store_true')
|
|
parser.add_argument('-q', '--quiet', help='print only errors',
|
|
action='store_true')
|
|
|
|
parser.add_argument('-h', '--help', action='help',
|
|
default=argparse.SUPPRESS,
|
|
help='show this help message and exit')
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.debug:
|
|
logger.setLevel(logging.DEBUG)
|
|
elif args.quiet:
|
|
logger.setLevel(logging.ERROR)
|
|
|
|
if args.url and args.filename:
|
|
logger.debug("--url and --filename detected. Ignoring --base-url "
|
|
"--release --installer-version arguments.")
|
|
filename = args.filename
|
|
buildtools_url = "%s/%s" % (args.url, filename)
|
|
else:
|
|
if args.base_url:
|
|
base_url = args.base_url
|
|
else:
|
|
base_url = DEFAULT_BASE_URL
|
|
if args.release:
|
|
# check if this is a pre-release "milestone" SDK
|
|
m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
|
|
args.release)
|
|
logger.debug("milestone regex: %s" % m)
|
|
if m and m.group('milestone'):
|
|
logger.debug("release[distro]: %s" % m.group('distro'))
|
|
logger.debug("release[version]: %s" % m.group('version'))
|
|
logger.debug("release[milestone]: %s" % m.group('milestone'))
|
|
if not args.build_date:
|
|
logger.error("Milestone installers require --build-date")
|
|
else:
|
|
if args.with_extended_buildtools:
|
|
filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
|
|
args.installer_version, args.build_date)
|
|
else:
|
|
filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % (
|
|
args.installer_version, args.build_date)
|
|
safe_filename = quote(filename)
|
|
buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
|
|
# regular release SDK
|
|
else:
|
|
if args.with_extended_buildtools:
|
|
filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version
|
|
else:
|
|
filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version
|
|
safe_filename = quote(filename)
|
|
buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
|
|
|
|
tmpsdk_dir = tempfile.mkdtemp()
|
|
try:
|
|
# Fetch installer
|
|
logger.info("Fetching buildtools installer")
|
|
tmpbuildtools = os.path.join(tmpsdk_dir, filename)
|
|
ret = subprocess.call("wget -q -O %s %s" %
|
|
(tmpbuildtools, buildtools_url), shell=True)
|
|
if ret != 0:
|
|
logger.error("Could not download file from %s" % buildtools_url)
|
|
return ret
|
|
|
|
# Verify checksum
|
|
if args.check:
|
|
logger.info("Fetching buildtools installer checksum")
|
|
checksum_type = ""
|
|
for checksum_type in ["md5sum", "sha256"]:
|
|
check_url = "{}.{}".format(buildtools_url, checksum_type)
|
|
checksum_filename = "{}.{}".format(filename, checksum_type)
|
|
tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
|
|
ret = subprocess.call("wget -q -O %s %s" %
|
|
(tmpbuildtools_checksum, check_url), shell=True)
|
|
if ret == 0:
|
|
break
|
|
else:
|
|
if ret != 0:
|
|
logger.error("Could not download file from %s" % check_url)
|
|
return ret
|
|
regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s\s(?P<path>.*/)?(?P<filename>.*)$")
|
|
with open(tmpbuildtools_checksum, 'rb') as f:
|
|
original = f.read()
|
|
m = re.search(regex, original.decode("utf-8"))
|
|
logger.debug("checksum regex match: %s" % m)
|
|
logger.debug("checksum: %s" % m.group('checksum'))
|
|
logger.debug("path: %s" % m.group('path'))
|
|
logger.debug("filename: %s" % m.group('filename'))
|
|
if filename != m.group('filename'):
|
|
logger.error("Filename does not match name in checksum")
|
|
return 1
|
|
checksum = m.group('checksum')
|
|
if checksum_type == "md5sum":
|
|
checksum_value = md5_file(tmpbuildtools)
|
|
else:
|
|
checksum_value = sha256_file(tmpbuildtools)
|
|
if checksum == checksum_value:
|
|
logger.info("Checksum success")
|
|
else:
|
|
logger.error("Checksum %s expected. Actual checksum is %s." %
|
|
(checksum, checksum_value))
|
|
|
|
# Make installer executable
|
|
logger.info("Making installer executable")
|
|
st = os.stat(tmpbuildtools)
|
|
os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
|
|
logger.debug(os.stat(tmpbuildtools))
|
|
if args.directory:
|
|
install_dir = args.directory
|
|
ret = subprocess.call("%s -d %s -y" %
|
|
(tmpbuildtools, install_dir), shell=True)
|
|
else:
|
|
install_dir = "/opt/poky/%s" % args.installer_version
|
|
ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
|
|
if ret != 0:
|
|
logger.error("Could not run buildtools installer")
|
|
|
|
# Setup the environment
|
|
logger.info("Setting up the environment")
|
|
regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
|
|
with open("%s/environment-setup-x86_64-pokysdk-linux" %
|
|
install_dir, 'rb') as f:
|
|
for line in f:
|
|
match = regex.search(line.decode('utf-8'))
|
|
logger.debug("export regex: %s" % match)
|
|
if match:
|
|
env_var = match.group('env_var')
|
|
logger.debug("env_var: %s" % env_var)
|
|
env_val = match.group('env_val')
|
|
logger.debug("env_val: %s" % env_val)
|
|
os.environ[env_var] = env_val
|
|
|
|
# Test installation
|
|
logger.info("Testing installation")
|
|
tool = ""
|
|
m = re.search("extended", tmpbuildtools)
|
|
logger.debug("extended regex: %s" % m)
|
|
if args.with_extended_buildtools and not m:
|
|
logger.info("Ignoring --with-extended-buildtools as filename "
|
|
"does not contain 'extended'")
|
|
if args.with_extended_buildtools and m:
|
|
tool = 'gcc'
|
|
else:
|
|
tool = 'tar'
|
|
logger.debug("install_dir: %s" % install_dir)
|
|
cmd = shlex.split("/usr/bin/which %s" % tool)
|
|
logger.debug("cmd: %s" % cmd)
|
|
logger.debug("tool: %s" % tool)
|
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
|
output, errors = proc.communicate()
|
|
logger.debug("proc.args: %s" % proc.args)
|
|
logger.debug("proc.communicate(): output %s" % output)
|
|
logger.debug("proc.communicate(): errors %s" % errors)
|
|
which_tool = output.decode('utf-8')
|
|
logger.debug("which %s: %s" % (tool, which_tool))
|
|
ret = proc.returncode
|
|
if not which_tool.startswith(install_dir):
|
|
logger.error("Something went wrong: %s not found in %s" %
|
|
(tool, install_dir))
|
|
if ret != 0:
|
|
logger.error("Something went wrong: installation failed")
|
|
else:
|
|
logger.info("Installation successful. Remember to source the "
|
|
"environment setup script now and in any new session.")
|
|
return ret
|
|
|
|
finally:
|
|
# cleanup tmp directory
|
|
shutil.rmtree(tmpsdk_dir)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
ret = main()
|
|
except Exception:
|
|
ret = 1
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
sys.exit(ret)
|