mirror of
https://git.yoctoproject.org/git/poky
synced 2026-01-01 13:58:04 +00:00
60d995ccc4
This patch adds support for extending the BB_ENV_PASSTHROUGH_ADDITIONS
environment variable from within the `init-build-env` wrapper script -
generated by either oe-core's oe-setup-build, or by `bitbake-setup` -
based on per-configuration JSON settings.
This enables CI workflows to inject environment-specific data - such
as build number, host, build type, or credentials required to fetch
from certain SRC_URIs - which cannot be captured via configuration
fragments alone. These variables are now handled early in the setup
process and exported directly into the build environment.
Example:
"bb-env-passthrough-additions": [
"ACME_DIR",
"ARTIFACTORY_TOKEN",
"ARTIFACTORY_USERNAME",
"GITHUB_TOKEN",
"GITHUB_PROTOCOL",
"KEY"
]
<snip>
the resulting 'init-build-env' would then be:
# environment passthrough added by bitbake-setup
export BB_ENV_PASSTHROUGH_ADDITIONS=" \
$BB_ENV_PASSTHROUGH_ADDITIONS \
ACME_DIR \
ARTIFACTORY_TOKEN \
ARTIFACTORY_USERNAME \
GITHUB_TOKEN \
GITHUB_PROTOCOL \
KEY"
# init-build-env wrapper created by bitbake-setup
. /tmp/acme_master-acme-distro_acme-machine_bang/layers/openembedded-core/oe-init-build-env /tmp/bitbake-setup/gs/acme_master-acme-distro_acme-machine_bang/build
(Bitbake rev: 782ab99e7a04fba43bdcf5763a6280785944ae3f)
Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
895 lines
43 KiB
Python
Executable File
895 lines
43 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import warnings
|
|
import json
|
|
import shutil
|
|
import time
|
|
import stat
|
|
import tempfile
|
|
import configparser
|
|
import datetime
|
|
import glob
|
|
import subprocess
|
|
|
|
default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")
|
|
|
|
bindir = os.path.abspath(os.path.dirname(__file__))
|
|
sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')]
|
|
|
|
import bb.msg
|
|
import bb.process
|
|
|
|
logger = bb.msg.logger_create('bitbake-setup', sys.stdout)
|
|
|
|
def cache_dir(top_dir):
|
|
return os.path.join(top_dir, '.bitbake-setup-cache')
|
|
|
|
def init_bb_cache(settings, args):
|
|
dldir = settings["default"]["dl-dir"]
|
|
bb_cachedir = os.path.join(cache_dir(args.top_dir), 'bitbake-cache')
|
|
|
|
d = bb.data.init()
|
|
d.setVar("DL_DIR", dldir)
|
|
d.setVar("BB_CACHEDIR", bb_cachedir)
|
|
d.setVar("__BBSRCREV_SEEN", "1")
|
|
if args.no_network:
|
|
d.setVar("BB_SRCREV_POLICY", "cache")
|
|
bb.fetch.fetcher_init(d)
|
|
return d
|
|
|
|
def save_bb_cache():
|
|
bb.fetch2.fetcher_parse_save()
|
|
bb.fetch2.fetcher_parse_done()
|
|
|
|
def get_config_name(config):
|
|
suffix = '.conf.json'
|
|
config_file = os.path.basename(config)
|
|
if config_file.endswith(suffix):
|
|
return config_file[:-len(suffix)]
|
|
else:
|
|
raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix))
|
|
|
|
def write_config(config, config_dir):
|
|
with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
|
|
json.dump(config, s, sort_keys=True, indent=4)
|
|
|
|
def commit_config(config_dir):
|
|
bb.process.run("git -C {} add .".format(config_dir))
|
|
bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime()))
|
|
|
|
def _write_layer_list(dest, repodirs):
|
|
layers = []
|
|
for r in repodirs:
|
|
for root, dirs, files in os.walk(os.path.join(dest,r)):
|
|
if os.path.basename(root) == 'conf' and 'layer.conf' in files:
|
|
layers.append(os.path.relpath(os.path.dirname(root), dest))
|
|
layers_f = os.path.join(dest, ".oe-layers.json")
|
|
with open(layers_f, 'w') as f:
|
|
json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)
|
|
|
|
def checkout_layers(layers, layerdir, d):
|
|
repodirs = []
|
|
oesetupbuild = None
|
|
print("Fetching layer/tool repositories into {}".format(layerdir))
|
|
for r_name in layers:
|
|
r_data = layers[r_name]
|
|
repodir = r_data["path"]
|
|
repodirs.append(repodir)
|
|
|
|
r_remote = r_data['git-remote']
|
|
rev = r_remote['rev']
|
|
branch = r_remote.get('branch', None)
|
|
remotes = r_remote['remotes']
|
|
|
|
for remote in remotes:
|
|
type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
|
|
fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
|
|
print(" {}".format(r_name))
|
|
if branch:
|
|
fetcher = bb.fetch.Fetch(["{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir)], d)
|
|
else:
|
|
fetcher = bb.fetch.Fetch(["{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir)], d)
|
|
do_fetch(fetcher, layerdir)
|
|
|
|
if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
|
|
oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
|
|
oeinitbuildenvdir = os.path.join(layerdir, repodir)
|
|
|
|
print(" ")
|
|
_write_layer_list(layerdir, repodirs)
|
|
|
|
if oesetupbuild:
|
|
links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'oe-init-build-env-dir': oeinitbuildenvdir}
|
|
for l,t in links.items():
|
|
symlink = os.path.join(layerdir, l)
|
|
if os.path.lexists(symlink):
|
|
os.remove(symlink)
|
|
os.symlink(os.path.relpath(t,layerdir),symlink)
|
|
|
|
def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
|
|
def _setup_build_conf(layers, build_conf_dir):
|
|
os.makedirs(build_conf_dir)
|
|
layers_s = []
|
|
for l in layers:
|
|
if l.startswith("{THISDIR}/"):
|
|
if thisdir:
|
|
l = l.format(THISDIR=thisdir)
|
|
else:
|
|
raise Exception("Configuration is using {THISDIR} to specify " \
|
|
"a layer path relative to itself. This can be done only " \
|
|
"when the configuration is specified by its path on local " \
|
|
"disk, not when it's in a registry or is fetched over http.")
|
|
if not os.path.isabs(l):
|
|
l = os.path.join(layerdir, l)
|
|
layers_s.append(" {} \\".format(l))
|
|
layers_s = "\n".join(layers_s)
|
|
bblayers_conf = """BBLAYERS ?= " \\
|
|
{}
|
|
"
|
|
""".format(layers_s)
|
|
with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f:
|
|
f.write(bblayers_conf)
|
|
|
|
local_conf = """#
|
|
# This file is intended for local configuration tweaks.
|
|
#
|
|
# If you would like to publish and share changes made to this file,
|
|
# it is recommended to put them into a distro config, or to create
|
|
# layer fragments from changes made here.
|
|
#
|
|
"""
|
|
with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f:
|
|
f.write(local_conf)
|
|
|
|
with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f:
|
|
f.write("")
|
|
|
|
with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f:
|
|
f.write(bitbake_config["description"] + "\n")
|
|
|
|
with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f:
|
|
f.write("")
|
|
|
|
def _make_init_build_env(builddir, oeinitbuildenvdir):
|
|
builddir = os.path.realpath(builddir)
|
|
cmd = "cd {}\nset {}\n. ./oe-init-build-env\n".format(oeinitbuildenvdir, builddir)
|
|
initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
|
|
|
|
with open(initbuild_in_builddir, 'w') as f:
|
|
f.write("# init-build-env wrapper created by bitbake-setup\n")
|
|
f.write(cmd + '\n')
|
|
|
|
def _prepend_passthrough_to_init_build_env(builddir):
|
|
env = bitbake_config.get("bb-env-passthrough-additions")
|
|
if not env:
|
|
return
|
|
|
|
initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
|
|
with open(initbuild_in_builddir) as f:
|
|
content = f.read()
|
|
|
|
joined = " \\\n".join(env)
|
|
env = "export BB_ENV_PASSTHROUGH_ADDITIONS=\" \\\n"
|
|
env += "${BB_ENV_PASSTHROUGH_ADDITIONS} \\\n"
|
|
env += joined
|
|
env += '"'
|
|
|
|
with open(initbuild_in_builddir, 'w') as f:
|
|
f.write("# environment passthrough added by bitbake-setup\n")
|
|
f.write(env + '\n')
|
|
f.write('\n')
|
|
f.write(content)
|
|
|
|
bitbake_builddir = os.path.join(builddir, "build")
|
|
print("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir))
|
|
|
|
template = bitbake_config.get("oe-template")
|
|
layers = bitbake_config.get("bb-layers")
|
|
if not template and not layers:
|
|
print("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.")
|
|
return
|
|
oesetupbuild = os.path.join(layerdir, 'setup-build')
|
|
if template and not os.path.exists(oesetupbuild):
|
|
raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template))
|
|
|
|
bitbake_confdir = os.path.join(bitbake_builddir, 'conf')
|
|
backup_bitbake_confdir = bitbake_confdir + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
if os.path.exists(bitbake_confdir):
|
|
os.rename(bitbake_confdir, backup_bitbake_confdir)
|
|
|
|
if layers:
|
|
_setup_build_conf(layers, bitbake_confdir)
|
|
|
|
if template:
|
|
bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir))
|
|
else:
|
|
oeinitbuildenvdir = os.path.join(layerdir, 'oe-init-build-env-dir')
|
|
if not os.path.exists(os.path.join(oeinitbuildenvdir, "oe-init-build-env")):
|
|
print("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment")
|
|
return
|
|
_make_init_build_env(bitbake_builddir, os.path.realpath(oeinitbuildenvdir))
|
|
|
|
_prepend_passthrough_to_init_build_env(bitbake_builddir)
|
|
|
|
siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
|
|
siteconf = os.path.normpath(os.path.join(builddir, '..', "site.conf"))
|
|
if os.path.lexists(siteconf_symlink):
|
|
os.remove(symlink)
|
|
os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)
|
|
|
|
|
|
init_script = os.path.join(bitbake_builddir, "init-build-env")
|
|
shell = "bash"
|
|
fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values())
|
|
if fragments:
|
|
bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments)))
|
|
|
|
if os.path.exists(backup_bitbake_confdir):
|
|
bitbake_config_diff = get_diff(backup_bitbake_confdir, bitbake_confdir)
|
|
if bitbake_config_diff:
|
|
print("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir))
|
|
print("The bitbake configuration has changed:")
|
|
print(bitbake_config_diff)
|
|
else:
|
|
shutil.rmtree(backup_bitbake_confdir)
|
|
|
|
print("This bitbake configuration provides:\n {}\n".format(bitbake_config["description"]))
|
|
|
|
readme = """{}\n\nAdditional information is in {} and {}\n
|
|
Source the environment using '. {}' to run builds from the command line.
|
|
The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf
|
|
""".format(
|
|
bitbake_config["description"],
|
|
os.path.join(bitbake_builddir,'conf/conf-summary.txt'),
|
|
os.path.join(bitbake_builddir,'conf/conf-notes.txt'),
|
|
init_script,
|
|
bitbake_builddir
|
|
)
|
|
readme_file = os.path.join(bitbake_builddir, "README")
|
|
with open(readme_file, 'w') as f:
|
|
f.write(readme)
|
|
print("Usage instructions and additional information are in\n {}\n".format(readme_file))
|
|
print("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n {}/conf\n".format(bitbake_builddir))
|
|
print("To run builds, source the environment using\n . {}".format(init_script))
|
|
|
|
def get_registry_config(registry_path, id):
|
|
for root, dirs, files in os.walk(registry_path):
|
|
for f in files:
|
|
if f.endswith('.conf.json') and id == get_config_name(f):
|
|
return os.path.join(root, f)
|
|
raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))
|
|
|
|
def update_build(config, confdir, builddir, layerdir, d):
|
|
layer_config = config["data"]["sources"]
|
|
layer_overrides = config["source-overrides"]["sources"]
|
|
for k,v in layer_overrides.items():
|
|
if k in layer_config:
|
|
layer_config[k]["git-remote"] = v["git-remote"]
|
|
checkout_layers(layer_config, layerdir, d)
|
|
bitbake_config = config["bitbake-config"]
|
|
thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
|
|
setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir)
|
|
|
|
def int_input(allowed_values):
|
|
n = None
|
|
while n is None:
|
|
try:
|
|
n = int(input())
|
|
except ValueError:
|
|
print('Not a valid number, please try again:')
|
|
continue
|
|
if n not in allowed_values:
|
|
print('Number {} not one of {}, please try again:'.format(n, allowed_values))
|
|
n = None
|
|
return n
|
|
|
|
def flatten_bitbake_configs(configs):
|
|
def merge_configs(c1,c2):
|
|
c_merged = {}
|
|
for k,v in c2.items():
|
|
if k not in c1.keys():
|
|
c_merged[k] = v
|
|
for k,v in c1.items():
|
|
if k not in c2.keys():
|
|
c_merged[k] = v
|
|
else:
|
|
c_merged[k] = c1[k] + c2[k]
|
|
del c_merged['configurations']
|
|
return c_merged
|
|
|
|
flattened_configs = []
|
|
for c in configs:
|
|
if 'configurations' not in c:
|
|
flattened_configs.append(c)
|
|
else:
|
|
for sub_c in flatten_bitbake_configs(c['configurations']):
|
|
flattened_configs.append(merge_configs(c, sub_c))
|
|
return flattened_configs
|
|
|
|
def choose_bitbake_config(configs, parameters, non_interactive):
|
|
flattened_configs = flatten_bitbake_configs(configs)
|
|
configs_dict = {i["name"]:i for i in flattened_configs}
|
|
|
|
if parameters:
|
|
config_id = parameters[0]
|
|
if config_id not in configs_dict:
|
|
raise Exception("Bitbake configuration {} not found; replace with one of {}".format(config_id, configs_dict))
|
|
return configs_dict[config_id]
|
|
|
|
enumerated_configs = list(enumerate(flattened_configs))
|
|
if len(enumerated_configs) == 1:
|
|
only_config = flattened_configs[0]
|
|
print("\nSelecting the only available bitbake configuration {}".format(only_config["name"]))
|
|
return only_config
|
|
|
|
if non_interactive:
|
|
raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict))
|
|
|
|
print("\nAvailable bitbake configurations:")
|
|
for n, config_data in enumerated_configs:
|
|
print("{}. {}\t{}".format(n, config_data["name"], config_data["description"]))
|
|
print("\nPlease select one of the above bitbake configurations by its number:")
|
|
config_n = int_input([i[0] for i in enumerated_configs])
|
|
return flattened_configs[config_n]
|
|
|
|
def choose_config(configs, non_interactive):
|
|
not_expired_configs = [k for k in configs.keys() if not has_expired(configs[k].get("expires", None))]
|
|
config_list = list(enumerate(not_expired_configs))
|
|
if len(config_list) == 1:
|
|
only_config = config_list[0][1]
|
|
print("\nSelecting the only available configuration {}\n".format(only_config))
|
|
return only_config
|
|
|
|
if non_interactive:
|
|
raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs))
|
|
|
|
print("\nAvailable configurations:")
|
|
for n, config_name in config_list:
|
|
config_data = configs[config_name]
|
|
expiry_date = config_data.get("expires", None)
|
|
config_desc = config_data["description"]
|
|
if expiry_date:
|
|
print("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date))
|
|
else:
|
|
print("{}. {}\t{}".format(n, config_name, config_desc))
|
|
print("\nPlease select one of the above configurations by its number:")
|
|
config_n = int_input([i[0] for i in config_list])
|
|
return config_list[config_n][1]
|
|
|
|
def choose_fragments(possibilities, parameters, non_interactive, skip_selection):
|
|
choices = {}
|
|
for k,v in possibilities.items():
|
|
if skip_selection and k in skip_selection:
|
|
print("Skipping a selection of {}, as requested on command line. The resulting bitbake configuration may require further manual adjustments.".format(k))
|
|
continue
|
|
choice = [o for o in v["options"] if o in parameters]
|
|
if len(choice) > 1:
|
|
raise Exception("Options specified on command line do not allow a single selection from possibilities {}, please remove one or more from {}".format(v["options"], parameters))
|
|
if len(choice) == 1:
|
|
choices[k] = choice[0]
|
|
continue
|
|
|
|
if non_interactive:
|
|
raise Exception("Unable to choose from options in non-interactive mode: {}".format(v["options"]))
|
|
|
|
print("\n" + v["description"] + ":")
|
|
options_enumerated = list(enumerate(v["options"]))
|
|
for n,o in options_enumerated:
|
|
print("{}. {}".format(n, o))
|
|
print("\nPlease select one of the above options by its number:")
|
|
option_n = int_input([i[0] for i in options_enumerated])
|
|
choices[k] = options_enumerated[option_n][1]
|
|
return choices
|
|
|
|
def obtain_config(settings, args, source_overrides, d):
|
|
if args.config:
|
|
config_id = args.config[0]
|
|
config_parameters = args.config[1:]
|
|
if os.path.exists(config_id):
|
|
print("Reading configuration from local file\n {}".format(config_id))
|
|
upstream_config = {'type':'local',
|
|
'path':os.path.abspath(config_id),
|
|
'name':get_config_name(config_id),
|
|
'data':json.load(open(config_id))
|
|
}
|
|
elif config_id.startswith("http://") or config_id.startswith("https://"):
|
|
print("Reading configuration from network URI\n {}".format(config_id))
|
|
import urllib.request
|
|
with urllib.request.urlopen(config_id) as f:
|
|
upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)}
|
|
else:
|
|
print("Looking up config {} in configuration registry".format(config_id))
|
|
registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d)
|
|
registry_configs = list_registry(registry_path, with_expired=True)
|
|
if config_id not in registry_configs:
|
|
raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id))
|
|
upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
|
|
expiry_date = upstream_config['data'].get("expires", None)
|
|
if has_expired(expiry_date):
|
|
print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date))
|
|
else:
|
|
registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d)
|
|
registry_configs = list_registry(registry_path, with_expired=True)
|
|
config_id = choose_config(registry_configs, args.non_interactive)
|
|
config_parameters = []
|
|
upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
|
|
|
|
upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive)
|
|
upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection)
|
|
upstream_config['non-interactive-cmdline-options'] = [config_id, upstream_config['bitbake-config']['name']] + sorted(upstream_config['bitbake-config']['oe-fragment-choices'].values())
|
|
upstream_config['source-overrides'] = source_overrides
|
|
upstream_config['skip-selection'] = args.skip_selection
|
|
return upstream_config
|
|
|
|
def init_config(settings, args, d):
|
|
stdout = sys.stdout
|
|
def handle_task_progress(event, d):
|
|
rate = event.rate if event.rate else ''
|
|
progress = event.progress if event.progress > 0 else 0
|
|
print("{}% {} ".format(progress, rate), file=stdout, end='\r')
|
|
|
|
source_overrides = json.load(open(args.source_overrides)) if args.source_overrides else {'sources':{}}
|
|
upstream_config = obtain_config(settings, args, source_overrides, d)
|
|
print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))
|
|
|
|
builddir = os.path.join(os.path.abspath(args.top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
|
|
if os.path.exists(os.path.join(builddir, "layers")):
|
|
print("Build already initialized in {}\nUse 'bitbake-setup status' to check if it needs to be updated or 'bitbake-setup update' to perform the update.".format(builddir))
|
|
return
|
|
|
|
print("Initializing a build in\n {}".format(builddir))
|
|
if not args.non_interactive:
|
|
y_or_n = input('Continue? (y/N): ')
|
|
if y_or_n != 'y':
|
|
exit()
|
|
print()
|
|
|
|
os.makedirs(builddir, exist_ok=True)
|
|
|
|
confdir = os.path.join(builddir, "config")
|
|
layerdir = os.path.join(builddir, "layers")
|
|
|
|
os.makedirs(confdir)
|
|
os.makedirs(layerdir)
|
|
|
|
bb.process.run("git -C {} init -b main".format(confdir))
|
|
# Make sure commiting doesn't fail if no default git user is configured on the machine
|
|
bb.process.run("git -C {} config user.name bitbake-setup".format(confdir))
|
|
bb.process.run("git -C {} config user.email bitbake-setup@not.set".format(confdir))
|
|
bb.process.run("git -C {} commit --no-verify --allow-empty -m 'Initial commit'".format(confdir))
|
|
|
|
bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)
|
|
|
|
write_config(upstream_config, confdir)
|
|
commit_config(confdir)
|
|
update_build(upstream_config, confdir, builddir, layerdir, d)
|
|
|
|
bb.event.remove("bb.build.TaskProgress", None)
|
|
|
|
def get_diff(file1, file2):
|
|
try:
|
|
bb.process.run('diff -uNr {} {}'.format(file1, file2))
|
|
except bb.process.ExecutionError as e:
|
|
if e.exitcode == 1:
|
|
return e.stdout
|
|
else:
|
|
raise e
|
|
return None
|
|
|
|
def are_layers_changed(layers, layerdir, d):
|
|
changed = False
|
|
for r_name in layers:
|
|
r_data = layers[r_name]
|
|
repodir = r_data["path"]
|
|
|
|
r_remote = r_data['git-remote']
|
|
rev = r_remote['rev']
|
|
branch = r_remote.get('branch', None)
|
|
remotes = r_remote['remotes']
|
|
|
|
for remote in remotes:
|
|
type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
|
|
fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
|
|
if branch:
|
|
fetcher = bb.fetch.FetchData("{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir), d)
|
|
else:
|
|
fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d)
|
|
upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default')
|
|
rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir)))
|
|
local_revision = rev_parse_result[0].strip()
|
|
if upstream_revision != local_revision:
|
|
changed = True
|
|
print('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remotes[remote]["uri"], os.path.join(layerdir, repodir), rev, local_revision, upstream_revision))
|
|
|
|
return changed
|
|
|
|
def build_status(settings, args, d, update=False):
|
|
builddir = args.build_dir
|
|
|
|
confdir = os.path.join(builddir, "config")
|
|
layerdir = os.path.join(builddir, "layers")
|
|
|
|
current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))
|
|
|
|
args.config = current_upstream_config['non-interactive-cmdline-options']
|
|
args.non_interactive = True
|
|
args.skip_selection = current_upstream_config['skip-selection']
|
|
source_overrides = current_upstream_config["source-overrides"]
|
|
new_upstream_config = obtain_config(settings, args, source_overrides, d)
|
|
|
|
write_config(new_upstream_config, confdir)
|
|
config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
|
|
|
|
if config_diff:
|
|
print('\nConfiguration in {} has changed:\n{}'.format(builddir, config_diff))
|
|
if update:
|
|
commit_config(confdir)
|
|
update_build(new_upstream_config, confdir, builddir, layerdir, d)
|
|
else:
|
|
bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
|
|
return
|
|
|
|
if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
|
|
if update:
|
|
update_build(current_upstream_config, confdir, builddir, layerdir, d)
|
|
return
|
|
|
|
print("\nConfiguration in {} has not changed.".format(builddir))
|
|
|
|
def build_update(settings, args, d):
|
|
build_status(settings, args, d, update=True)
|
|
|
|
def do_fetch(fetcher, dir):
|
|
# git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch
|
|
# and we need to set up smth similar here
|
|
fetchlogdir = os.path.join(dir, 'logs')
|
|
os.makedirs(fetchlogdir, exist_ok=True)
|
|
fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
|
|
with open(fetchlog, 'a') as f:
|
|
oldstdout = sys.stdout
|
|
sys.stdout = f
|
|
fetcher.download()
|
|
fetcher.unpack(dir)
|
|
sys.stdout = oldstdout
|
|
|
|
def update_registry(registry, cachedir, d):
|
|
registrydir = 'configurations'
|
|
if registry.startswith("."):
|
|
full_registrydir = os.path.join(os.getcwd(), registry, registrydir)
|
|
elif registry.startswith("/"):
|
|
full_registrydir = os.path.join(registry, registrydir)
|
|
else:
|
|
full_registrydir = os.path.join(cachedir, registrydir)
|
|
print("Fetching configuration registry\n {}\ninto\n {}".format(registry, full_registrydir))
|
|
fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d)
|
|
do_fetch(fetcher, cachedir)
|
|
return full_registrydir
|
|
|
|
def has_expired(expiry_date):
|
|
if expiry_date:
|
|
return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date)
|
|
return False
|
|
|
|
def list_registry(registry_path, with_expired):
|
|
json_data = {}
|
|
|
|
for root, dirs, files in os.walk(registry_path):
|
|
for f in files:
|
|
if f.endswith('.conf.json'):
|
|
config_name = get_config_name(f)
|
|
config_data = json.load(open(os.path.join(root, f)))
|
|
config_desc = config_data["description"]
|
|
expiry_date = config_data.get("expires", None)
|
|
if expiry_date:
|
|
if with_expired or not has_expired(expiry_date):
|
|
json_data[config_name] = {"description": config_desc, "expires": expiry_date}
|
|
else:
|
|
json_data[config_name] = {"description": config_desc}
|
|
return json_data
|
|
|
|
def list_configs(settings, args, d):
|
|
registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d)
|
|
json_data = list_registry(registry_path, args.with_expired)
|
|
print("\nAvailable configurations:")
|
|
for config_name, config_data in json_data.items():
|
|
expiry_date = config_data.get("expires", None)
|
|
config_desc = config_data["description"]
|
|
if expiry_date:
|
|
if args.with_expired or not has_expired(expiry_date):
|
|
print("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date))
|
|
else:
|
|
print("{}\t{}".format(config_name, config_desc))
|
|
print("\nRun 'init' with one of the above configuration identifiers to set up a build.")
|
|
|
|
if args.write_json:
|
|
with open(args.write_json, 'w') as f:
|
|
json.dump(json_data, f, sort_keys=True, indent=4)
|
|
print("Available configurations written into {}".format(args.write_json))
|
|
|
|
def install_buildtools(settings, args, d):
|
|
buildtools_install_dir = os.path.join(args.build_dir, 'buildtools')
|
|
if os.path.exists(buildtools_install_dir):
|
|
if not args.force:
|
|
print("Buildtools are already installed in {}.".format(buildtools_install_dir))
|
|
env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*'))
|
|
if env_scripts:
|
|
print("If you wish to use them, you need to source the environment setup script e.g.")
|
|
for s in env_scripts:
|
|
print("$ . {}".format(s))
|
|
print("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.")
|
|
return
|
|
shutil.rmtree(buildtools_install_dir)
|
|
|
|
install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools')
|
|
buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
|
|
print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
|
|
subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)
|
|
|
|
def default_settings_path(top_dir):
|
|
return os.path.join(top_dir, 'bitbake-setup.conf')
|
|
|
|
def write_settings(top_dir, force_replace, non_interactive=True):
|
|
settings_path = default_settings_path(top_dir)
|
|
if not os.path.exists(settings_path) or force_replace:
|
|
|
|
settings = configparser.ConfigParser()
|
|
settings['default'] = {
|
|
'registry':default_registry,
|
|
'dl-dir':os.path.join(top_dir, '.bitbake-setup-downloads'),
|
|
}
|
|
os.makedirs(os.path.dirname(settings_path), exist_ok=True)
|
|
|
|
siteconfpath = os.path.join(top_dir, 'site.conf')
|
|
print('Configuration registry set to\n {}\n'.format(settings['default']['registry']))
|
|
print('Bitbake-setup download cache (DL_DIR) set to\n {}\n'.format(settings['default']['dl-dir']))
|
|
print('A new settings file will be created in\n {}\n'.format(settings_path))
|
|
print('A common site.conf file will be created, please edit or replace before running builds\n {}\n'.format(siteconfpath))
|
|
if not non_interactive:
|
|
y_or_n = input('Bitbake-setup will be configured with the above settings in {}, (y/N): '.format(top_dir))
|
|
if y_or_n != 'y':
|
|
print("\nYou can run 'bitbake-setup install-settings' to edit them before setting up builds")
|
|
exit()
|
|
print()
|
|
|
|
if os.path.exists(settings_path):
|
|
backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
os.rename(settings_path, backup_conf)
|
|
print("Previous settings are in {}".format(backup_conf))
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
|
|
if os.path.exists(siteconfpath):
|
|
backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
os.rename(siteconfpath, backup_siteconf)
|
|
print("Previous settings are in {}".format(backup_siteconf))
|
|
with open(siteconfpath, 'w') as siteconffile:
|
|
siteconffile.write('# This file is intended for build host-specific bitbake settings\n')
|
|
|
|
def load_settings(top_dir, non_interactive):
|
|
# This creates a new settings file if it does not yet exist
|
|
write_settings(top_dir, force_replace=False, non_interactive=non_interactive)
|
|
|
|
settings_path = default_settings_path(top_dir)
|
|
settings = configparser.ConfigParser()
|
|
print('Loading settings from\n {}\n'.format(settings_path))
|
|
settings.read_file(open(settings_path))
|
|
return settings
|
|
|
|
def global_settings_path(args):
|
|
return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'config')
|
|
|
|
def write_global_settings(settings_path, force_replace, non_interactive=True):
|
|
if not os.path.exists(settings_path) or force_replace:
|
|
|
|
settings = configparser.ConfigParser()
|
|
settings['default'] = {
|
|
'top-dir-prefix':os.path.expanduser('~'),
|
|
'top-dir-name':'bitbake-builds'
|
|
}
|
|
os.makedirs(os.path.dirname(settings_path), exist_ok=True)
|
|
print('Configuring global settings in\n {}\n'.format(settings_path))
|
|
print('Top directory prefix (where all top level directories are created) set to\n {}\n'.format(settings['default']['top-dir-prefix']))
|
|
print('Top directory name (this is added to the top directory prefix to form a top directory where builds are set up) set to\n {}\n'.format(settings['default']['top-dir-name']))
|
|
if not non_interactive:
|
|
y_or_n = input('Write out the global settings as specified above (y/N)? ')
|
|
if y_or_n != 'y':
|
|
print("\nYou can run 'bitbake-setup install-global-settings' to edit them before setting up builds")
|
|
exit()
|
|
print()
|
|
|
|
if os.path.exists(settings_path):
|
|
backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
os.rename(settings_path, backup_conf)
|
|
print("Previous global settings are in {}".format(backup_conf))
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
|
|
def load_global_settings(settings_path, non_interactive):
|
|
# This creates a new settings file if it does not yet exist
|
|
write_global_settings(settings_path, force_replace=False, non_interactive=non_interactive)
|
|
|
|
settings = configparser.ConfigParser()
|
|
print('Loading global settings from\n {}\n'.format(settings_path))
|
|
settings.read_file(open(settings_path))
|
|
return settings
|
|
|
|
def change_settings(top_dir, new_settings):
|
|
settings = load_settings(top_dir, non_interactive=True)
|
|
for section, section_settings in new_settings.items():
|
|
for setting, value in section_settings.items():
|
|
settings[section][setting] = value
|
|
print("Setting '{}' in section '{}' is changed to '{}'".format(setting, section, value))
|
|
|
|
settings_path = default_settings_path(top_dir)
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
print("New settings written to {}".format(settings_path))
|
|
return settings
|
|
|
|
def change_global_settings(settings_path, new_settings):
|
|
settings = load_global_settings(settings_path, non_interactive=True)
|
|
for section, section_settings in new_settings.items():
|
|
for setting, value in section_settings.items():
|
|
settings[section][setting] = value
|
|
print("Setting '{}' in section '{}' is changed to '{}'".format(setting, section, value))
|
|
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
print("New global settings written to {}".format(settings_path))
|
|
return settings
|
|
|
|
def get_build_dir_via_bbpath():
|
|
bbpath = os.environ.get('BBPATH')
|
|
if bbpath:
|
|
bitbake_dir = os.path.normpath(bbpath.split(':')[0])
|
|
if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
|
|
build_dir = os.path.dirname(bitbake_dir)
|
|
return build_dir
|
|
return None
|
|
|
|
def get_top_dir(args, global_settings):
|
|
build_dir_via_bbpath = get_build_dir_via_bbpath()
|
|
if build_dir_via_bbpath:
|
|
top_dir = os.path.dirname(build_dir_via_bbpath)
|
|
if os.path.exists(default_settings_path(top_dir)):
|
|
return top_dir
|
|
|
|
if hasattr(args, 'build_dir'):
|
|
# commands without --top-dir-prefix/name arguments (status, update) still need to know where
|
|
# the top dir is, but it should be auto-deduced as parent of args.build_dir
|
|
top_dir = os.path.dirname(os.path.normpath(args.build_dir))
|
|
return top_dir
|
|
|
|
top_dir_prefix = args.top_dir_prefix if args.top_dir_prefix else global_settings['default']['top-dir-prefix']
|
|
top_dir_name = args.top_dir_name if args.top_dir_name else global_settings['default']['top-dir-name']
|
|
return os.path.join(top_dir_prefix, top_dir_name)
|
|
|
|
def main():
|
|
def add_top_dir_arg(parser):
|
|
parser.add_argument('--top-dir-prefix', help='Top level directory prefix. This is where all top level directories are created.')
|
|
parser.add_argument('--top-dir-name', help='Top level directory name. Together with the top directory prefix this forms a top directory where builds are set up and downloaded configurations and layers are cached for reproducibility and offline builds.')
|
|
|
|
def add_build_dir_arg(parser):
|
|
build_dir = get_build_dir_via_bbpath()
|
|
if build_dir:
|
|
parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH")
|
|
else:
|
|
parser.add_argument('--build-dir', required=True, help="Path to the build")
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="BitBake setup utility. Run with 'init' argument to get started.",
|
|
epilog="Use %(prog)s <subcommand> --help to get help on a specific command"
|
|
)
|
|
parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
|
|
parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
|
|
parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
|
|
parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.')
|
|
parser.add_argument('--global-settings', action='store', help='Path to the global settings file where defaults for top directory prefix and name can be specified')
|
|
|
|
subparsers = parser.add_subparsers()
|
|
|
|
parser_list = subparsers.add_parser('list', help='List available configurations')
|
|
add_top_dir_arg(parser_list)
|
|
parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.')
|
|
parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
|
|
parser_list.set_defaults(func=list_configs)
|
|
|
|
parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it')
|
|
add_top_dir_arg(parser_init)
|
|
parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
|
|
parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
|
|
parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
|
|
parser_init.add_argument('--build-dir-name', action='store', help='A custom build directory name under the top directory.')
|
|
parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
|
|
parser_init.set_defaults(func=init_config)
|
|
|
|
parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration')
|
|
add_build_dir_arg(parser_status)
|
|
parser_status.set_defaults(func=build_status)
|
|
|
|
parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration')
|
|
add_build_dir_arg(parser_update)
|
|
parser_update.set_defaults(func=build_update)
|
|
|
|
parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
|
|
add_build_dir_arg(parser_install_buildtools)
|
|
parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
|
|
parser_install_buildtools.set_defaults(func=install_buildtools)
|
|
|
|
parser_install_settings = subparsers.add_parser('install-settings', help='Write a settings file with default values into the top level directory (contains the location of build configuration registry, downloads directory and other settings specific to a top directory)')
|
|
add_top_dir_arg(parser_install_settings)
|
|
parser_install_settings.set_defaults(func=write_settings)
|
|
|
|
parser_install_global_settings = subparsers.add_parser('install-global-settings', help='Write a global settings file with default values (contains the default prefix and name of the top directory)')
|
|
parser_install_global_settings.set_defaults(func=write_global_settings)
|
|
|
|
parser_change_setting = subparsers.add_parser('change-setting', help='Change a setting in the settings file')
|
|
add_top_dir_arg(parser_change_setting)
|
|
parser_change_setting.add_argument('section', help="Section in a settings file, typically 'default'")
|
|
parser_change_setting.add_argument('key', help="Name of the setting")
|
|
parser_change_setting.add_argument('value', help="Value of the setting")
|
|
parser_change_setting.set_defaults(func=change_settings)
|
|
|
|
parser_change_global_setting = subparsers.add_parser('change-global-setting', help='Change a setting in the global settings file')
|
|
parser_change_global_setting.add_argument('section', help="Section in a global settings file, typically 'default'")
|
|
parser_change_global_setting.add_argument('key', help="Name of the setting")
|
|
parser_change_global_setting.add_argument('value', help="Value of the setting")
|
|
parser_change_global_setting.set_defaults(func=change_global_settings)
|
|
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(stream=sys.stdout)
|
|
if args.debug:
|
|
logger.setLevel(logging.DEBUG)
|
|
elif args.quiet:
|
|
logger.setLevel(logging.ERROR)
|
|
|
|
# Need to re-run logger_create with color argument
|
|
# (will be the same logger since it has the same name)
|
|
bb.msg.logger_create('bitbake-setup', output=sys.stdout,
|
|
color=args.color,
|
|
level=logger.getEffectiveLevel())
|
|
|
|
if 'func' in args:
|
|
if args.func == write_global_settings:
|
|
write_global_settings(global_settings_path(args), force_replace=True)
|
|
return
|
|
elif args.func == change_global_settings:
|
|
change_global_settings(global_settings_path(args), {args.section:{args.key:args.value}})
|
|
return
|
|
|
|
if hasattr(args, 'build_dir'):
|
|
if not os.path.exists(os.path.join(args.build_dir,'build', 'init-build-env')):
|
|
print("Not a valid build directory: build/init-build-env does not exist in {}".format(args.build_dir))
|
|
return
|
|
|
|
if not hasattr(args, 'non_interactive'):
|
|
args.non_interactive = True
|
|
|
|
global_settings = load_global_settings(global_settings_path(args), args.non_interactive)
|
|
args.top_dir = get_top_dir(args, global_settings)
|
|
|
|
print('Bitbake-setup is using {} as top directory (can be changed with --top-dir-prefix/name arguments or by setting them in {}).\n'.format(args.top_dir, global_settings_path(args)))
|
|
if args.func == write_settings:
|
|
write_settings(args.top_dir, force_replace=True)
|
|
elif args.func == change_settings:
|
|
change_settings(args.top_dir, {args.section:{args.key:args.value}})
|
|
else:
|
|
settings = load_settings(args.top_dir, args.non_interactive)
|
|
d = init_bb_cache(settings, args)
|
|
args.func(settings, args, d)
|
|
save_bb_cache()
|
|
else:
|
|
from argparse import Namespace
|
|
parser.print_help()
|
|
|
|
main()
|