mirror of
https://git.yoctoproject.org/git/poky
synced 2026-01-01 13:58:04 +00:00
401e7b6a10
This is the outcome of various discussions, suggestions and pull
requests on github.
What has specifically changed?
1. The sources for the settings are no longer separated, but are stacked and given priorities,
from highest to lowest:
a. '--setting section key value' on the command line
b. a settings file in the top directory
c. a global settings file in ~/.config/bitbake-setup/ (or in a file pointed to by --global-settings)
Any setting can be in any of these three locations (other than top dir name and prefix which do not
make sense in the settings file in the top directory).
2. A global settings file must contain all of the needed settings, while a settings file
in the top directory can be empty (and this is how they are written out if they do not exist).
Specifically, both dl-dir and registry settings have been relocated to the global file,
and dl-dir defaults to ~/.cache/bitbake-setup/downloads, rather than somewhere in top dir.
3. The file name for both global and top dir settings is now 'settings.conf'.
4. --top-dir-prefix and --top-dir-name options have been removed and superseded by
a generic, universal --setting option.
5. 'install-settings' command has been removed, as it is no longer does anything useful,
and is superseded by the 'setting' command (see below).
'install-global-settings' has been retained, to be able to have a set of global defaults
that can be changed without initializing a build.
6. 'change-setting', 'change-global-setting' and 'install-settings' have all been replaced
by a single 'setting' command that mimics 'git config' in its parameters:
a. Changing a setting: bitbake-setup setting [--global] default dl-dir /path/to/downloads
b. Removing a setting: bitbake-setup setting [--global] --unset default dl-dir
(Bitbake rev: 713e7f213c6d4a620be9ce34d5f4396af48e1d69)
Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
888 lines
42 KiB
Python
Executable File
888 lines
42 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import warnings
|
|
import json
|
|
import shutil
|
|
import time
|
|
import stat
|
|
import tempfile
|
|
import configparser
|
|
import datetime
|
|
import glob
|
|
import subprocess
|
|
|
|
default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")
|
|
|
|
bindir = os.path.abspath(os.path.dirname(__file__))
|
|
sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')]
|
|
|
|
import bb.msg
|
|
import bb.process
|
|
|
|
logger = bb.msg.logger_create('bitbake-setup', sys.stdout)
|
|
|
|
def cache_dir(top_dir):
|
|
return os.path.join(top_dir, '.bitbake-setup-cache')
|
|
|
|
def init_bb_cache(top_dir, settings, args):
|
|
dldir = settings["default"]["dl-dir"]
|
|
bb_cachedir = os.path.join(cache_dir(top_dir), 'bitbake-cache')
|
|
|
|
d = bb.data.init()
|
|
d.setVar("DL_DIR", dldir)
|
|
d.setVar("BB_CACHEDIR", bb_cachedir)
|
|
d.setVar("__BBSRCREV_SEEN", "1")
|
|
if args.no_network:
|
|
d.setVar("BB_SRCREV_POLICY", "cache")
|
|
bb.fetch.fetcher_init(d)
|
|
return d
|
|
|
|
def save_bb_cache():
|
|
bb.fetch2.fetcher_parse_save()
|
|
bb.fetch2.fetcher_parse_done()
|
|
|
|
def get_config_name(config):
|
|
suffix = '.conf.json'
|
|
config_file = os.path.basename(config)
|
|
if config_file.endswith(suffix):
|
|
return config_file[:-len(suffix)]
|
|
else:
|
|
raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix))
|
|
|
|
def write_config(config, config_dir):
|
|
with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
|
|
json.dump(config, s, sort_keys=True, indent=4)
|
|
|
|
def commit_config(config_dir):
|
|
bb.process.run("git -C {} add .".format(config_dir))
|
|
bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime()))
|
|
|
|
def _write_layer_list(dest, repodirs):
|
|
layers = []
|
|
for r in repodirs:
|
|
for root, dirs, files in os.walk(os.path.join(dest,r)):
|
|
if os.path.basename(root) == 'conf' and 'layer.conf' in files:
|
|
layers.append(os.path.relpath(os.path.dirname(root), dest))
|
|
layers_f = os.path.join(dest, ".oe-layers.json")
|
|
with open(layers_f, 'w') as f:
|
|
json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)
|
|
|
|
def checkout_layers(layers, layerdir, d):
|
|
repodirs = []
|
|
oesetupbuild = None
|
|
print("Fetching layer/tool repositories into {}".format(layerdir))
|
|
for r_name in layers:
|
|
r_data = layers[r_name]
|
|
repodir = r_data["path"]
|
|
repodirs.append(repodir)
|
|
|
|
r_remote = r_data['git-remote']
|
|
rev = r_remote['rev']
|
|
branch = r_remote.get('branch', None)
|
|
remotes = r_remote['remotes']
|
|
|
|
for remote in remotes:
|
|
type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
|
|
fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
|
|
print(" {}".format(r_name))
|
|
if branch:
|
|
fetcher = bb.fetch.Fetch(["{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir)], d)
|
|
else:
|
|
fetcher = bb.fetch.Fetch(["{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir)], d)
|
|
do_fetch(fetcher, layerdir)
|
|
|
|
if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
|
|
oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
|
|
oeinitbuildenvdir = os.path.join(layerdir, repodir)
|
|
|
|
print(" ")
|
|
_write_layer_list(layerdir, repodirs)
|
|
|
|
if oesetupbuild:
|
|
links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'oe-init-build-env-dir': oeinitbuildenvdir}
|
|
for l,t in links.items():
|
|
symlink = os.path.join(layerdir, l)
|
|
if os.path.lexists(symlink):
|
|
os.remove(symlink)
|
|
os.symlink(os.path.relpath(t,layerdir),symlink)
|
|
|
|
def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
|
|
def _setup_build_conf(layers, build_conf_dir):
|
|
os.makedirs(build_conf_dir)
|
|
layers_s = []
|
|
for l in layers:
|
|
if l.startswith("{THISDIR}/"):
|
|
if thisdir:
|
|
l = l.format(THISDIR=thisdir)
|
|
else:
|
|
raise Exception("Configuration is using {THISDIR} to specify " \
|
|
"a layer path relative to itself. This can be done only " \
|
|
"when the configuration is specified by its path on local " \
|
|
"disk, not when it's in a registry or is fetched over http.")
|
|
if not os.path.isabs(l):
|
|
l = os.path.join(layerdir, l)
|
|
layers_s.append(" {} \\".format(l))
|
|
layers_s = "\n".join(layers_s)
|
|
bblayers_conf = """BBLAYERS ?= " \\
|
|
{}
|
|
"
|
|
""".format(layers_s)
|
|
with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f:
|
|
f.write(bblayers_conf)
|
|
|
|
local_conf = """#
|
|
# This file is intended for local configuration tweaks.
|
|
#
|
|
# If you would like to publish and share changes made to this file,
|
|
# it is recommended to put them into a distro config, or to create
|
|
# layer fragments from changes made here.
|
|
#
|
|
"""
|
|
with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f:
|
|
f.write(local_conf)
|
|
|
|
with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f:
|
|
f.write("")
|
|
|
|
with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f:
|
|
f.write(bitbake_config["description"] + "\n")
|
|
|
|
with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f:
|
|
f.write("")
|
|
|
|
def _make_init_build_env(builddir, oeinitbuildenvdir):
|
|
builddir = os.path.realpath(builddir)
|
|
cmd = "cd {}\nset {}\n. ./oe-init-build-env\n".format(oeinitbuildenvdir, builddir)
|
|
initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
|
|
|
|
with open(initbuild_in_builddir, 'w') as f:
|
|
f.write("# init-build-env wrapper created by bitbake-setup\n")
|
|
f.write(cmd + '\n')
|
|
|
|
def _prepend_passthrough_to_init_build_env(builddir):
|
|
env = bitbake_config.get("bb-env-passthrough-additions")
|
|
if not env:
|
|
return
|
|
|
|
initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
|
|
with open(initbuild_in_builddir) as f:
|
|
content = f.read()
|
|
|
|
joined = " \\\n".join(env)
|
|
env = "export BB_ENV_PASSTHROUGH_ADDITIONS=\" \\\n"
|
|
env += "${BB_ENV_PASSTHROUGH_ADDITIONS} \\\n"
|
|
env += joined
|
|
env += '"'
|
|
|
|
with open(initbuild_in_builddir, 'w') as f:
|
|
f.write("# environment passthrough added by bitbake-setup\n")
|
|
f.write(env + '\n')
|
|
f.write('\n')
|
|
f.write(content)
|
|
|
|
bitbake_builddir = os.path.join(builddir, "build")
|
|
print("Setting up bitbake configuration in\n {}\n".format(bitbake_builddir))
|
|
|
|
template = bitbake_config.get("oe-template")
|
|
layers = bitbake_config.get("bb-layers")
|
|
if not template and not layers:
|
|
print("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.")
|
|
return
|
|
oesetupbuild = os.path.join(layerdir, 'setup-build')
|
|
if template and not os.path.exists(oesetupbuild):
|
|
raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template))
|
|
|
|
bitbake_confdir = os.path.join(bitbake_builddir, 'conf')
|
|
backup_bitbake_confdir = bitbake_confdir + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
if os.path.exists(bitbake_confdir):
|
|
os.rename(bitbake_confdir, backup_bitbake_confdir)
|
|
|
|
if layers:
|
|
_setup_build_conf(layers, bitbake_confdir)
|
|
|
|
if template:
|
|
bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir))
|
|
else:
|
|
oeinitbuildenvdir = os.path.join(layerdir, 'oe-init-build-env-dir')
|
|
if not os.path.exists(os.path.join(oeinitbuildenvdir, "oe-init-build-env")):
|
|
print("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment")
|
|
return
|
|
_make_init_build_env(bitbake_builddir, os.path.realpath(oeinitbuildenvdir))
|
|
|
|
_prepend_passthrough_to_init_build_env(bitbake_builddir)
|
|
|
|
siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
|
|
siteconf = os.path.normpath(os.path.join(builddir, '..', "site.conf"))
|
|
if os.path.lexists(siteconf_symlink):
|
|
os.remove(symlink)
|
|
os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)
|
|
|
|
|
|
init_script = os.path.join(bitbake_builddir, "init-build-env")
|
|
shell = "bash"
|
|
fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values())
|
|
if fragments:
|
|
bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments)))
|
|
|
|
if os.path.exists(backup_bitbake_confdir):
|
|
bitbake_config_diff = get_diff(backup_bitbake_confdir, bitbake_confdir)
|
|
if bitbake_config_diff:
|
|
print("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir))
|
|
print("The bitbake configuration has changed:")
|
|
print(bitbake_config_diff)
|
|
else:
|
|
shutil.rmtree(backup_bitbake_confdir)
|
|
|
|
print("This bitbake configuration provides:\n {}\n".format(bitbake_config["description"]))
|
|
|
|
readme = """{}\n\nAdditional information is in {} and {}\n
|
|
Source the environment using '. {}' to run builds from the command line.
|
|
The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf
|
|
""".format(
|
|
bitbake_config["description"],
|
|
os.path.join(bitbake_builddir,'conf/conf-summary.txt'),
|
|
os.path.join(bitbake_builddir,'conf/conf-notes.txt'),
|
|
init_script,
|
|
bitbake_builddir
|
|
)
|
|
readme_file = os.path.join(bitbake_builddir, "README")
|
|
with open(readme_file, 'w') as f:
|
|
f.write(readme)
|
|
print("Usage instructions and additional information are in\n {}\n".format(readme_file))
|
|
print("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n {}/conf\n".format(bitbake_builddir))
|
|
print("To run builds, source the environment using\n . {}".format(init_script))
|
|
|
|
def get_registry_config(registry_path, id):
|
|
for root, dirs, files in os.walk(registry_path):
|
|
for f in files:
|
|
if f.endswith('.conf.json') and id == get_config_name(f):
|
|
return os.path.join(root, f)
|
|
raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))
|
|
|
|
def update_build(config, confdir, builddir, layerdir, d):
|
|
layer_config = config["data"]["sources"]
|
|
layer_overrides = config["source-overrides"]["sources"]
|
|
for k,v in layer_overrides.items():
|
|
if k in layer_config:
|
|
layer_config[k]["git-remote"] = v["git-remote"]
|
|
checkout_layers(layer_config, layerdir, d)
|
|
bitbake_config = config["bitbake-config"]
|
|
thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
|
|
setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir)
|
|
|
|
def int_input(allowed_values):
|
|
n = None
|
|
while n is None:
|
|
try:
|
|
n = int(input())
|
|
except ValueError:
|
|
print('Not a valid number, please try again:')
|
|
continue
|
|
if n not in allowed_values:
|
|
print('Number {} not one of {}, please try again:'.format(n, allowed_values))
|
|
n = None
|
|
return n
|
|
|
|
def flatten_bitbake_configs(configs):
|
|
def merge_configs(c1,c2):
|
|
c_merged = {}
|
|
for k,v in c2.items():
|
|
if k not in c1.keys():
|
|
c_merged[k] = v
|
|
for k,v in c1.items():
|
|
if k not in c2.keys():
|
|
c_merged[k] = v
|
|
else:
|
|
c_merged[k] = c1[k] + c2[k]
|
|
del c_merged['configurations']
|
|
return c_merged
|
|
|
|
flattened_configs = []
|
|
for c in configs:
|
|
if 'configurations' not in c:
|
|
flattened_configs.append(c)
|
|
else:
|
|
for sub_c in flatten_bitbake_configs(c['configurations']):
|
|
flattened_configs.append(merge_configs(c, sub_c))
|
|
return flattened_configs
|
|
|
|
def choose_bitbake_config(configs, parameters, non_interactive):
|
|
flattened_configs = flatten_bitbake_configs(configs)
|
|
configs_dict = {i["name"]:i for i in flattened_configs}
|
|
|
|
if parameters:
|
|
config_id = parameters[0]
|
|
if config_id not in configs_dict:
|
|
raise Exception("Bitbake configuration {} not found; replace with one of {}".format(config_id, configs_dict))
|
|
return configs_dict[config_id]
|
|
|
|
enumerated_configs = list(enumerate(flattened_configs))
|
|
if len(enumerated_configs) == 1:
|
|
only_config = flattened_configs[0]
|
|
print("\nSelecting the only available bitbake configuration {}".format(only_config["name"]))
|
|
return only_config
|
|
|
|
if non_interactive:
|
|
raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict))
|
|
|
|
print("\nAvailable bitbake configurations:")
|
|
for n, config_data in enumerated_configs:
|
|
print("{}. {}\t{}".format(n, config_data["name"], config_data["description"]))
|
|
print("\nPlease select one of the above bitbake configurations by its number:")
|
|
config_n = int_input([i[0] for i in enumerated_configs])
|
|
return flattened_configs[config_n]
|
|
|
|
def choose_config(configs, non_interactive):
|
|
not_expired_configs = [k for k in configs.keys() if not has_expired(configs[k].get("expires", None))]
|
|
config_list = list(enumerate(not_expired_configs))
|
|
if len(config_list) == 1:
|
|
only_config = config_list[0][1]
|
|
print("\nSelecting the only available configuration {}\n".format(only_config))
|
|
return only_config
|
|
|
|
if non_interactive:
|
|
raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs))
|
|
|
|
print("\nAvailable configurations:")
|
|
for n, config_name in config_list:
|
|
config_data = configs[config_name]
|
|
expiry_date = config_data.get("expires", None)
|
|
config_desc = config_data["description"]
|
|
if expiry_date:
|
|
print("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date))
|
|
else:
|
|
print("{}. {}\t{}".format(n, config_name, config_desc))
|
|
print("\nPlease select one of the above configurations by its number:")
|
|
config_n = int_input([i[0] for i in config_list])
|
|
return config_list[config_n][1]
|
|
|
|
def choose_fragments(possibilities, parameters, non_interactive, skip_selection):
|
|
choices = {}
|
|
for k,v in possibilities.items():
|
|
if skip_selection and k in skip_selection:
|
|
print("Skipping a selection of {}, as requested on command line. The resulting bitbake configuration may require further manual adjustments.".format(k))
|
|
continue
|
|
choice = [o for o in v["options"] if o in parameters]
|
|
if len(choice) > 1:
|
|
raise Exception("Options specified on command line do not allow a single selection from possibilities {}, please remove one or more from {}".format(v["options"], parameters))
|
|
if len(choice) == 1:
|
|
choices[k] = choice[0]
|
|
continue
|
|
|
|
if non_interactive:
|
|
raise Exception("Unable to choose from options in non-interactive mode: {}".format(v["options"]))
|
|
|
|
print("\n" + v["description"] + ":")
|
|
options_enumerated = list(enumerate(v["options"]))
|
|
for n,o in options_enumerated:
|
|
print("{}. {}".format(n, o))
|
|
print("\nPlease select one of the above options by its number:")
|
|
option_n = int_input([i[0] for i in options_enumerated])
|
|
choices[k] = options_enumerated[option_n][1]
|
|
return choices
|
|
|
|
def obtain_config(top_dir, settings, args, source_overrides, d):
|
|
if args.config:
|
|
config_id = args.config[0]
|
|
config_parameters = args.config[1:]
|
|
if os.path.exists(config_id):
|
|
print("Reading configuration from local file\n {}".format(config_id))
|
|
upstream_config = {'type':'local',
|
|
'path':os.path.abspath(config_id),
|
|
'name':get_config_name(config_id),
|
|
'data':json.load(open(config_id))
|
|
}
|
|
elif config_id.startswith("http://") or config_id.startswith("https://"):
|
|
print("Reading configuration from network URI\n {}".format(config_id))
|
|
import urllib.request
|
|
with urllib.request.urlopen(config_id) as f:
|
|
upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)}
|
|
else:
|
|
print("Looking up config {} in configuration registry".format(config_id))
|
|
registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
|
|
registry_configs = list_registry(registry_path, with_expired=True)
|
|
if config_id not in registry_configs:
|
|
raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id))
|
|
upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
|
|
expiry_date = upstream_config['data'].get("expires", None)
|
|
if has_expired(expiry_date):
|
|
print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date))
|
|
else:
|
|
registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
|
|
registry_configs = list_registry(registry_path, with_expired=True)
|
|
config_id = choose_config(registry_configs, args.non_interactive)
|
|
config_parameters = []
|
|
upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
|
|
|
|
upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive)
|
|
upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection)
|
|
upstream_config['non-interactive-cmdline-options'] = [config_id, upstream_config['bitbake-config']['name']] + sorted(upstream_config['bitbake-config']['oe-fragment-choices'].values())
|
|
upstream_config['source-overrides'] = source_overrides
|
|
upstream_config['skip-selection'] = args.skip_selection
|
|
return upstream_config
|
|
|
|
def init_config(top_dir, settings, args, d):
|
|
stdout = sys.stdout
|
|
def handle_task_progress(event, d):
|
|
rate = event.rate if event.rate else ''
|
|
progress = event.progress if event.progress > 0 else 0
|
|
print("{}% {} ".format(progress, rate), file=stdout, end='\r')
|
|
|
|
source_overrides = json.load(open(args.source_overrides)) if args.source_overrides else {'sources':{}}
|
|
upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
|
|
print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))
|
|
|
|
builddir = os.path.join(os.path.abspath(top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
|
|
if os.path.exists(os.path.join(builddir, "layers")):
|
|
print("Build already initialized in {}\nUse 'bitbake-setup status' to check if it needs to be updated or 'bitbake-setup update' to perform the update.".format(builddir))
|
|
return
|
|
|
|
print("Initializing a build in\n {}".format(builddir))
|
|
if not args.non_interactive:
|
|
y_or_n = input('Continue? (y/N): ')
|
|
if y_or_n != 'y':
|
|
exit()
|
|
print()
|
|
|
|
os.makedirs(builddir, exist_ok=True)
|
|
|
|
confdir = os.path.join(builddir, "config")
|
|
layerdir = os.path.join(builddir, "layers")
|
|
|
|
os.makedirs(confdir)
|
|
os.makedirs(layerdir)
|
|
|
|
bb.process.run("git -C {} init -b main".format(confdir))
|
|
# Make sure commiting doesn't fail if no default git user is configured on the machine
|
|
bb.process.run("git -C {} config user.name bitbake-setup".format(confdir))
|
|
bb.process.run("git -C {} config user.email bitbake-setup@not.set".format(confdir))
|
|
bb.process.run("git -C {} commit --no-verify --allow-empty -m 'Initial commit'".format(confdir))
|
|
|
|
bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)
|
|
|
|
write_config(upstream_config, confdir)
|
|
commit_config(confdir)
|
|
update_build(upstream_config, confdir, builddir, layerdir, d)
|
|
|
|
bb.event.remove("bb.build.TaskProgress", None)
|
|
|
|
def get_diff(file1, file2):
|
|
try:
|
|
bb.process.run('diff -uNr {} {}'.format(file1, file2))
|
|
except bb.process.ExecutionError as e:
|
|
if e.exitcode == 1:
|
|
return e.stdout
|
|
else:
|
|
raise e
|
|
return None
|
|
|
|
def are_layers_changed(layers, layerdir, d):
|
|
changed = False
|
|
for r_name in layers:
|
|
r_data = layers[r_name]
|
|
repodir = r_data["path"]
|
|
|
|
r_remote = r_data['git-remote']
|
|
rev = r_remote['rev']
|
|
branch = r_remote.get('branch', None)
|
|
remotes = r_remote['remotes']
|
|
|
|
for remote in remotes:
|
|
type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
|
|
fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
|
|
if branch:
|
|
fetcher = bb.fetch.FetchData("{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir), d)
|
|
else:
|
|
fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d)
|
|
upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default')
|
|
rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir)))
|
|
local_revision = rev_parse_result[0].strip()
|
|
if upstream_revision != local_revision:
|
|
changed = True
|
|
print('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remotes[remote]["uri"], os.path.join(layerdir, repodir), rev, local_revision, upstream_revision))
|
|
|
|
return changed
|
|
|
|
def build_status(top_dir, settings, args, d, update=False):
|
|
builddir = args.build_dir
|
|
|
|
confdir = os.path.join(builddir, "config")
|
|
layerdir = os.path.join(builddir, "layers")
|
|
|
|
current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))
|
|
|
|
args.config = current_upstream_config['non-interactive-cmdline-options']
|
|
args.non_interactive = True
|
|
args.skip_selection = current_upstream_config['skip-selection']
|
|
source_overrides = current_upstream_config["source-overrides"]
|
|
new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
|
|
|
|
write_config(new_upstream_config, confdir)
|
|
config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
|
|
|
|
if config_diff:
|
|
print('\nConfiguration in {} has changed:\n{}'.format(builddir, config_diff))
|
|
if update:
|
|
commit_config(confdir)
|
|
update_build(new_upstream_config, confdir, builddir, layerdir, d)
|
|
else:
|
|
bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
|
|
return
|
|
|
|
if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
|
|
if update:
|
|
update_build(current_upstream_config, confdir, builddir, layerdir, d)
|
|
return
|
|
|
|
print("\nConfiguration in {} has not changed.".format(builddir))
|
|
|
|
def build_update(top_dir, settings, args, d):
|
|
build_status(top_dir, settings, args, d, update=True)
|
|
|
|
def do_fetch(fetcher, dir):
|
|
# git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch
|
|
# and we need to set up smth similar here
|
|
fetchlogdir = os.path.join(dir, 'logs')
|
|
os.makedirs(fetchlogdir, exist_ok=True)
|
|
fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
|
|
with open(fetchlog, 'a') as f:
|
|
oldstdout = sys.stdout
|
|
sys.stdout = f
|
|
fetcher.download()
|
|
fetcher.unpack(dir)
|
|
sys.stdout = oldstdout
|
|
|
|
def update_registry(registry, cachedir, d):
|
|
registrydir = 'configurations'
|
|
if registry.startswith("."):
|
|
full_registrydir = os.path.join(os.getcwd(), registry, registrydir)
|
|
elif registry.startswith("/"):
|
|
full_registrydir = os.path.join(registry, registrydir)
|
|
else:
|
|
full_registrydir = os.path.join(cachedir, registrydir)
|
|
print("Fetching configuration registry\n {}\ninto\n {}".format(registry, full_registrydir))
|
|
fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d)
|
|
do_fetch(fetcher, cachedir)
|
|
return full_registrydir
|
|
|
|
def has_expired(expiry_date):
|
|
if expiry_date:
|
|
return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date)
|
|
return False
|
|
|
|
def list_registry(registry_path, with_expired):
|
|
json_data = {}
|
|
|
|
for root, dirs, files in os.walk(registry_path):
|
|
for f in files:
|
|
if f.endswith('.conf.json'):
|
|
config_name = get_config_name(f)
|
|
config_data = json.load(open(os.path.join(root, f)))
|
|
config_desc = config_data["description"]
|
|
expiry_date = config_data.get("expires", None)
|
|
if expiry_date:
|
|
if with_expired or not has_expired(expiry_date):
|
|
json_data[config_name] = {"description": config_desc, "expires": expiry_date}
|
|
else:
|
|
json_data[config_name] = {"description": config_desc}
|
|
return json_data
|
|
|
|
def list_configs(top_dir, settings, args, d):
|
|
registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
|
|
json_data = list_registry(registry_path, args.with_expired)
|
|
print("\nAvailable configurations:")
|
|
for config_name, config_data in json_data.items():
|
|
expiry_date = config_data.get("expires", None)
|
|
config_desc = config_data["description"]
|
|
if expiry_date:
|
|
if args.with_expired or not has_expired(expiry_date):
|
|
print("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date))
|
|
else:
|
|
print("{}\t{}".format(config_name, config_desc))
|
|
print("\nRun 'init' with one of the above configuration identifiers to set up a build.")
|
|
|
|
if args.write_json:
|
|
with open(args.write_json, 'w') as f:
|
|
json.dump(json_data, f, sort_keys=True, indent=4)
|
|
print("Available configurations written into {}".format(args.write_json))
|
|
|
|
def install_buildtools(top_dir, settings, args, d):
|
|
buildtools_install_dir = os.path.join(args.build_dir, 'buildtools')
|
|
if os.path.exists(buildtools_install_dir):
|
|
if not args.force:
|
|
print("Buildtools are already installed in {}.".format(buildtools_install_dir))
|
|
env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*'))
|
|
if env_scripts:
|
|
print("If you wish to use them, you need to source the environment setup script e.g.")
|
|
for s in env_scripts:
|
|
print("$ . {}".format(s))
|
|
print("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.")
|
|
return
|
|
shutil.rmtree(buildtools_install_dir)
|
|
|
|
install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools')
|
|
buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
|
|
print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
|
|
subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)
|
|
|
|
def default_settings_path(top_dir):
|
|
return os.path.join(top_dir, 'settings.conf')
|
|
|
|
def create_settings(top_dir, non_interactive=True):
|
|
settings_path = default_settings_path(top_dir)
|
|
settings = configparser.ConfigParser()
|
|
settings['default'] = {
|
|
}
|
|
os.makedirs(os.path.dirname(settings_path), exist_ok=True)
|
|
|
|
siteconfpath = os.path.join(top_dir, 'site.conf')
|
|
print('A new empty settings file will be created in (you can add settings to it to override defaults from the global settings file)\n {}\n'.format(settings_path))
|
|
print('A common site.conf file will be created, please edit or replace before running builds\n {}\n'.format(siteconfpath))
|
|
if not non_interactive:
|
|
y_or_n = input('Bitbake-setup will be configured with the above settings in {}, (y/N): '.format(top_dir))
|
|
if y_or_n != 'y':
|
|
print("\nYou can run 'bitbake-setup install-settings' to edit them before setting up builds")
|
|
exit()
|
|
print()
|
|
|
|
if os.path.exists(settings_path):
|
|
backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
os.rename(settings_path, backup_conf)
|
|
print("Previous settings are in {}".format(backup_conf))
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
|
|
if os.path.exists(siteconfpath):
|
|
backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
os.rename(siteconfpath, backup_siteconf)
|
|
print("Previous settings are in {}".format(backup_siteconf))
|
|
with open(siteconfpath, 'w') as siteconffile:
|
|
siteconffile.write('# This file is intended for build host-specific bitbake settings\n')
|
|
|
|
def load_settings(top_dir, non_interactive):
|
|
settings_path = default_settings_path(top_dir)
|
|
if not os.path.exists(settings_path):
|
|
create_settings(top_dir, non_interactive=non_interactive)
|
|
|
|
settings = configparser.ConfigParser()
|
|
print('Loading settings from\n {}\n'.format(settings_path))
|
|
settings.read_file(open(settings_path))
|
|
return settings
|
|
|
|
def global_settings_path(args):
|
|
return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf')
|
|
|
|
def create_global_settings(settings_path, non_interactive=True):
|
|
settings = configparser.ConfigParser()
|
|
settings['default'] = {
|
|
'top-dir-prefix':os.path.expanduser('~'),
|
|
'top-dir-name':'bitbake-builds',
|
|
'registry':default_registry,
|
|
'dl-dir':os.path.join(os.path.expanduser('~'), '.cache', 'bitbake-setup', 'downloads'),
|
|
}
|
|
os.makedirs(os.path.dirname(settings_path), exist_ok=True)
|
|
print('Configuring global settings in\n {}\n'.format(settings_path))
|
|
print('Top directory prefix (where all top level directories are created) set to\n {}\n'.format(settings['default']['top-dir-prefix']))
|
|
print('Top directory name (this is added to the top directory prefix to form a top directory where builds are set up) set to\n {}\n'.format(settings['default']['top-dir-name']))
|
|
print('Configuration registry set to\n {}\n'.format(settings['default']['registry']))
|
|
print('Bitbake-setup download cache (DL_DIR) set to\n {}\n'.format(settings['default']['dl-dir']))
|
|
if not non_interactive:
|
|
y_or_n = input('Write out the global settings as specified above (y/N)? ')
|
|
if y_or_n != 'y':
|
|
print("\nYou can run 'bitbake-setup install-global-settings' to edit them before setting up builds")
|
|
exit()
|
|
print()
|
|
|
|
if os.path.exists(settings_path):
|
|
backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
|
|
os.rename(settings_path, backup_conf)
|
|
print("Previous global settings are in {}".format(backup_conf))
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
|
|
def load_global_settings(settings_path, non_interactive):
|
|
if not os.path.exists(settings_path):
|
|
create_global_settings(settings_path, non_interactive=non_interactive)
|
|
|
|
settings = configparser.ConfigParser()
|
|
print('Loading global settings from\n {}\n'.format(settings_path))
|
|
settings.read_file(open(settings_path))
|
|
return settings
|
|
|
|
def change_setting(settings_path, settings, args):
|
|
if args.section and args.key and args.value:
|
|
settings[args.section][args.key] = args.value
|
|
print("Setting '{}' in section '{}' is changed to '{}'".format(args.key, args.section, args.value))
|
|
if args.unset:
|
|
section = args.unset[0]
|
|
setting = args.unset[1]
|
|
if section in settings.keys() and setting in settings[section].keys():
|
|
del settings[section][setting]
|
|
print("Setting '{} in section '{}' is removed".format(setting, section))
|
|
|
|
with open(settings_path, 'w') as settingsfile:
|
|
settings.write(settingsfile)
|
|
print("New settings written to {}".format(settings_path))
|
|
|
|
def setting_global(args):
|
|
settings = load_global_settings(global_settings_path(args), args.non_interactive)
|
|
settings_path = global_settings_path(args)
|
|
change_setting(settings_path, settings, args)
|
|
|
|
def setting(top_dir, args):
|
|
settings = load_settings(top_dir, args.non_interactive)
|
|
settings_path = default_settings_path(top_dir)
|
|
change_setting(settings_path, settings, args)
|
|
|
|
def get_build_dir_via_bbpath():
|
|
bbpath = os.environ.get('BBPATH')
|
|
if bbpath:
|
|
bitbake_dir = os.path.normpath(bbpath.split(':')[0])
|
|
if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
|
|
build_dir = os.path.dirname(bitbake_dir)
|
|
return build_dir
|
|
return None
|
|
|
|
def get_top_dir(args, settings):
|
|
build_dir_via_bbpath = get_build_dir_via_bbpath()
|
|
if build_dir_via_bbpath:
|
|
top_dir = os.path.dirname(build_dir_via_bbpath)
|
|
if os.path.exists(default_settings_path(top_dir)):
|
|
return top_dir
|
|
|
|
if hasattr(args, 'build_dir'):
|
|
top_dir = os.path.dirname(os.path.normpath(args.build_dir))
|
|
return top_dir
|
|
|
|
top_dir_prefix = settings['default']['top-dir-prefix']
|
|
top_dir_name = settings['default']['top-dir-name']
|
|
return os.path.join(top_dir_prefix, top_dir_name)
|
|
|
|
def merge_settings(global_settings, local_settings, cmdline_settings):
|
|
all_settings = global_settings
|
|
for section, section_settings in local_settings.items():
|
|
for setting, value in section_settings.items():
|
|
all_settings[section][setting] = value
|
|
|
|
for (section, setting, value) in cmdline_settings:
|
|
all_settings[section][setting] = value
|
|
|
|
return all_settings
|
|
|
|
def main():
|
|
def add_build_dir_arg(parser):
|
|
build_dir = get_build_dir_via_bbpath()
|
|
if build_dir:
|
|
parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH")
|
|
else:
|
|
parser.add_argument('--build-dir', required=True, help="Path to the build")
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="BitBake setup utility. Run with 'init' argument to get started.",
|
|
epilog="Use %(prog)s <subcommand> --help to get help on a specific command"
|
|
)
|
|
parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
|
|
parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
|
|
parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
|
|
parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.')
|
|
parser.add_argument('--global-settings', action='store', help='Path to the global settings file.')
|
|
parser.add_argument('--setting', default=[], action='append', nargs=3, help='Modify a setting (for this bitbake-setup invocation only), for example "--setting default top-dir-prefix /path/to/top/dir".')
|
|
|
|
subparsers = parser.add_subparsers()
|
|
|
|
parser_list = subparsers.add_parser('list', help='List available configurations')
|
|
parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.')
|
|
parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
|
|
parser_list.set_defaults(func=list_configs)
|
|
|
|
parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it')
|
|
parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
|
|
parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
|
|
parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
|
|
parser_init.add_argument('--build-dir-name', action='store', help='A custom build directory name under the top directory.')
|
|
parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
|
|
parser_init.set_defaults(func=init_config)
|
|
|
|
parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration')
|
|
add_build_dir_arg(parser_status)
|
|
parser_status.set_defaults(func=build_status)
|
|
|
|
parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration')
|
|
add_build_dir_arg(parser_update)
|
|
parser_update.set_defaults(func=build_update)
|
|
|
|
parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
|
|
add_build_dir_arg(parser_install_buildtools)
|
|
parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
|
|
parser_install_buildtools.set_defaults(func=install_buildtools)
|
|
|
|
parser_install_global_settings = subparsers.add_parser('install-global-settings', help='Write a global settings file with default values')
|
|
parser_install_global_settings.set_defaults(func=create_global_settings)
|
|
|
|
parser_setting = subparsers.add_parser('setting', help='Set or unset a setting in a setting file (e.g. the default prefix and name of the top directory, the location of build configuration registry, downloads directory and other settings specific to a top directory)')
|
|
parser_setting.add_argument('section', nargs='?', help="Section in a settings file, typically 'default'")
|
|
parser_setting.add_argument('key', nargs='?', help="Name of the setting")
|
|
parser_setting.add_argument('value', nargs='?', help="Value of the setting")
|
|
parser_setting.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory")
|
|
parser_setting.add_argument('--unset', nargs=2, help="Unset a setting, e.g. 'bitbake-setup setting --unset default registry' would revert to the registry setting in a global settings file")
|
|
parser_setting.set_defaults(func=setting)
|
|
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(stream=sys.stdout)
|
|
if args.debug:
|
|
logger.setLevel(logging.DEBUG)
|
|
elif args.quiet:
|
|
logger.setLevel(logging.ERROR)
|
|
|
|
# Need to re-run logger_create with color argument
|
|
# (will be the same logger since it has the same name)
|
|
bb.msg.logger_create('bitbake-setup', output=sys.stdout,
|
|
color=args.color,
|
|
level=logger.getEffectiveLevel())
|
|
|
|
if 'func' in args:
|
|
if args.func == create_global_settings:
|
|
create_global_settings(global_settings_path(args))
|
|
return
|
|
|
|
if hasattr(args, 'build_dir'):
|
|
if not os.path.exists(os.path.join(args.build_dir,'build', 'init-build-env')):
|
|
print("Not a valid build directory: build/init-build-env does not exist in {}".format(args.build_dir))
|
|
return
|
|
|
|
if not hasattr(args, 'non_interactive'):
|
|
args.non_interactive = True
|
|
|
|
if args.func == setting and vars(args)['global']:
|
|
setting_global(args)
|
|
return
|
|
|
|
global_settings = load_global_settings(global_settings_path(args), args.non_interactive)
|
|
top_dir = get_top_dir(args, merge_settings(global_settings, {}, args.setting))
|
|
|
|
if args.func == setting:
|
|
setting(top_dir, args)
|
|
return
|
|
|
|
print('Bitbake-setup is using {} as top directory (can be changed by setting top dir prefix and name in {}).\n'.format(top_dir, global_settings_path(args)))
|
|
|
|
settings = load_settings(top_dir, args.non_interactive)
|
|
settings = merge_settings(global_settings, settings, args.setting)
|
|
d = init_bb_cache(top_dir, settings, args)
|
|
args.func(top_dir, settings, args, d)
|
|
save_bb_cache()
|
|
else:
|
|
from argparse import Namespace
|
|
parser.print_help()
|
|
|
|
main()
|