poky/bitbake/bin/bitbake-setup

#!/usr/bin/env python3

#
# SPDX-License-Identifier: GPL-2.0-only
#

import logging
import os
import sys
import argparse
import warnings
import json
import shutil
import time
import stat
import tempfile
import configparser
import datetime
import glob
import subprocess
import copy

default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")

bindir = os.path.abspath(os.path.dirname(__file__))
sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')]

import bb.msg
import bb.process

logger = bb.msg.logger_create('bitbake-setup', sys.stdout)

def cache_dir(top_dir):
    return os.path.join(top_dir, '.bitbake-setup-cache')

def init_bb_cache(top_dir, settings, args):
    dldir = settings["default"]["dl-dir"]
    bb_cachedir = os.path.join(cache_dir(top_dir), 'bitbake-cache')

    d = bb.data.init()
    d.setVar("DL_DIR", dldir)
    d.setVar("BB_CACHEDIR", bb_cachedir)
    d.setVar("__BBSRCREV_SEEN", "1")
    if args.no_network:
        d.setVar("BB_SRCREV_POLICY", "cache")
    bb.fetch.fetcher_init(d)
    return d

def save_bb_cache():
    bb.fetch2.fetcher_parse_save()
    bb.fetch2.fetcher_parse_done()

def get_config_name(config):
    suffix = '.conf.json'
    config_file = os.path.basename(config)
    if config_file.endswith(suffix):
        return config_file[:-len(suffix)]
    else:
        raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix))

def write_upstream_config(config_dir, config_data):
    with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
        json.dump(config_data, s, sort_keys=True, indent=4)

def write_sources_fixed_revisions(config_dir, config_data):
    sources = {}
    sources['sources'] = config_data
    with open(os.path.join(config_dir, "sources-fixed-revisions.json"),'w') as s:
        json.dump(sources, s, sort_keys=True, indent=4)

def commit_config(config_dir):
    bb.process.run("git -C {} add .".format(config_dir))
    bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime()))

def _write_layer_list(dest, repodirs):
    layers = []
    for r in repodirs:
        for root, dirs, files in os.walk(os.path.join(dest,r)):
            if os.path.basename(root) == 'conf' and 'layer.conf' in files:
                layers.append(os.path.relpath(os.path.dirname(root), dest))
    layers_f = os.path.join(dest, ".oe-layers.json")
    with open(layers_f, 'w') as f:
        json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)

def checkout_layers(layers, layerdir, d):
    layers_fixed_revisions = copy.deepcopy(layers)
    repodirs = []
    oesetupbuild = None
    print("Fetching layer/tool repositories into {}".format(layerdir))
    for r_name in layers:
        r_data = layers[r_name]
        repodir = r_data["path"]
        repodirs.append(repodir)

        r_remote = r_data['git-remote']
        rev = r_remote['rev']
        branch = r_remote.get('branch', None)
        remotes = r_remote['remotes']

        for remote in remotes:
            prot,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
            fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
            print("    {}".format(r_name))
            if branch:
                src_uri = f"{fetchuri};protocol={prot};rev={rev};branch={branch};destsuffix={repodir}"
            else:
                src_uri = f"{fetchuri};protocol={prot};rev={rev};nobranch=1;destsuffix={repodir}"
            fetcher = bb.fetch.Fetch([src_uri], d)
            do_fetch(fetcher, layerdir)
            urldata = fetcher.ud[src_uri]
            revision = urldata.revision
            layers_fixed_revisions[r_name]['git-remote']['rev'] = revision

        if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
            oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
            oeinitbuildenvdir = os.path.join(layerdir, repodir)

    print("        ")
    _write_layer_list(layerdir, repodirs)

    if oesetupbuild:
        links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'oe-init-build-env-dir': oeinitbuildenvdir}
        for l,t in links.items():
            symlink = os.path.join(layerdir, l)
            if os.path.lexists(symlink):
                os.remove(symlink)
            os.symlink(os.path.relpath(t,layerdir),symlink)

    return layers_fixed_revisions

def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir):
    def _setup_build_conf(layers, relative_layers, build_conf_dir):
        os.makedirs(build_conf_dir)
        layers_s = []

        for l in layers:
            l = os.path.join(layerdir, l)
            layers_s.append("  {} \\".format(l))

        for l in relative_layers:
            if thisdir:
                l = os.path.join(thisdir, l)
            else:
                raise Exception("Configuration is using bb-layers-relative to specify " \
                "a layer path relative to itself. This can be done only " \
                "when the configuration is specified by its path on local " \
                "disk, not when it's in a registry or is fetched over http.")
            layers_s.append("  {} \\".format(l))

        layers_s = "\n".join(layers_s)
        bblayers_conf = """BBLAYERS ?= " \\
{}
  "
""".format(layers_s)
        with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f:
            f.write(bblayers_conf)

        local_conf = """#
# This file is intended for local configuration tweaks.
#
# If you would like to publish and share changes made to this file,
# it is recommended to put them into a distro config, or to create
# layer fragments from changes made here.
#
"""
        with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f:
            f.write(local_conf)

        with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f:
            f.write("")

        with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f:
            f.write(bitbake_config["description"] + "\n")

        with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f:
            f.write("")

    def _make_init_build_env(builddir, oeinitbuildenvdir):
        builddir = os.path.realpath(builddir)
        cmd = "cd {}\nset {}\n. ./oe-init-build-env\n".format(oeinitbuildenvdir, builddir)
        initbuild_in_builddir = os.path.join(builddir, 'init-build-env')

        with open(initbuild_in_builddir, 'w') as f:
            f.write("# init-build-env wrapper created by bitbake-setup\n")
            f.write(cmd + '\n')

    def _prepend_passthrough_to_init_build_env(builddir):
        env = bitbake_config.get("bb-env-passthrough-additions")
        if not env:
            return

        initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
        with open(initbuild_in_builddir) as f:
            content = f.read()

        joined = " \\\n".join(env)
        env = "export BB_ENV_PASSTHROUGH_ADDITIONS=\" \\\n"
        env += "${BB_ENV_PASSTHROUGH_ADDITIONS} \\\n"
        env += joined
        env += '"'

        with open(initbuild_in_builddir, 'w') as f:
            f.write("# environment passthrough added by bitbake-setup\n")
            f.write(env + '\n')
            f.write('\n')
            f.write(content)

    bitbake_builddir = os.path.join(setupdir, "build")
    print("Setting up bitbake configuration in\n    {}\n".format(bitbake_builddir))

    template = bitbake_config.get("oe-template")
    layers = bitbake_config.get("bb-layers")
    if not template and not layers:
        print("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.")
        return
    oesetupbuild = os.path.join(layerdir, 'setup-build')
    if template and not os.path.exists(oesetupbuild):
        raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template))

    bitbake_confdir = os.path.join(bitbake_builddir, 'conf')
    backup_bitbake_confdir = bitbake_confdir + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
    if os.path.exists(bitbake_confdir):
        os.rename(bitbake_confdir, backup_bitbake_confdir)

    if layers:
        relative_layers = bitbake_config.get("bb-layers-relative") or []
        _setup_build_conf(layers, relative_layers, bitbake_confdir)

    if template:
        bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir))
    else:
        oeinitbuildenvdir = os.path.join(layerdir, 'oe-init-build-env-dir')
        if not os.path.exists(os.path.join(oeinitbuildenvdir, "oe-init-build-env")):
            print("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment")
            return
        _make_init_build_env(bitbake_builddir, os.path.realpath(oeinitbuildenvdir))

    _prepend_passthrough_to_init_build_env(bitbake_builddir)

    siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
    siteconf = os.path.normpath(os.path.join(setupdir, '..', "site.conf"))
    if os.path.lexists(siteconf_symlink):
        os.remove(symlink)
    os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)


    init_script = os.path.join(bitbake_builddir, "init-build-env")
    shell = "bash"
    fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values())
    if fragments:
        bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments)))

    if os.path.exists(backup_bitbake_confdir):
        bitbake_config_diff = get_diff(backup_bitbake_confdir, bitbake_confdir)
        if bitbake_config_diff:
            print("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir))
            print("The bitbake configuration has changed:")
            print(bitbake_config_diff)
        else:
            shutil.rmtree(backup_bitbake_confdir)

    print("This bitbake configuration provides:\n    {}\n".format(bitbake_config["description"]))

    readme = """{}\n\nAdditional information is in {} and {}\n
Source the environment using '. {}' to run builds from the command line.
The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf
""".format(
        bitbake_config["description"],
        os.path.join(bitbake_builddir,'conf/conf-summary.txt'),
        os.path.join(bitbake_builddir,'conf/conf-notes.txt'),
        init_script,
        bitbake_builddir
        )
    readme_file = os.path.join(bitbake_builddir, "README")
    with open(readme_file, 'w') as f:
        f.write(readme)
    print("Usage instructions and additional information are in\n     {}\n".format(readme_file))
    print("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n    {}/conf\n".format(bitbake_builddir))
    print("To run builds, source the environment using\n    . {}".format(init_script))

def get_registry_config(registry_path, id):
    for root, dirs, files in os.walk(registry_path):
        for f in files:
            if f.endswith('.conf.json') and id == get_config_name(f):
                return os.path.join(root, f)
    raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))

def update_build(config, confdir, setupdir, layerdir, d):
    layer_config = copy.deepcopy(config["data"]["sources"])
    layer_overrides = config["source-overrides"]["sources"]
    for k,v in layer_overrides.items():
        if k in layer_config:
            layer_config[k]["git-remote"] = v["git-remote"]
    sources_fixed_revisions = checkout_layers(layer_config, layerdir, d)
    bitbake_config = config["bitbake-config"]
    thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
    setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir)
    write_sources_fixed_revisions(confdir, sources_fixed_revisions)

def int_input(allowed_values):
    n = None
    while n is None:
        try:
            n = int(input())
        except ValueError:
            print('Not a valid number, please try again:')
            continue
        if n not in allowed_values:
            print('Number {} not one of {}, please try again:'.format(n, allowed_values))
            n = None
    return n

def flatten_bitbake_configs(configs):
    def merge_configs(c1,c2):
        c_merged = {}
        for k,v in c2.items():
            if k not in c1.keys():
                c_merged[k] = v
        for k,v in c1.items():
            if k not in c2.keys():
                c_merged[k] = v
            else:
               c_merged[k] = c1[k] + c2[k]
        del c_merged['configurations']
        return c_merged

    flattened_configs = []
    for c in configs:
        if 'configurations' not in c:
            flattened_configs.append(c)
        else:
            for sub_c in flatten_bitbake_configs(c['configurations']):
                flattened_configs.append(merge_configs(c, sub_c))
    return flattened_configs

def choose_bitbake_config(configs, parameters, non_interactive):
    flattened_configs = flatten_bitbake_configs(configs)
    configs_dict = {i["name"]:i for i in flattened_configs}

    if parameters:
        config_id = parameters[0]
        if config_id not in configs_dict:
            raise Exception("Bitbake configuration {} not found; replace with one of {}".format(config_id, configs_dict))
        return configs_dict[config_id]

    enumerated_configs = list(enumerate(flattened_configs))
    if len(enumerated_configs) == 1:
        only_config = flattened_configs[0]
        print("\nSelecting the only available bitbake configuration {}".format(only_config["name"]))
        return only_config

    if non_interactive:
        raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict))

    print("\nAvailable bitbake configurations:")
    for n, config_data in enumerated_configs:
        print("{}. {}\t{}".format(n, config_data["name"], config_data["description"]))
    print("\nPlease select one of the above bitbake configurations by its number:")
    config_n = int_input([i[0] for i in enumerated_configs])
    return flattened_configs[config_n]

def choose_config(configs, non_interactive):
    not_expired_configs = [k for k in configs.keys() if not has_expired(configs[k].get("expires", None))]
    config_list = list(enumerate(not_expired_configs))
    if len(config_list) == 1:
        only_config = config_list[0][1]
        print("\nSelecting the only available configuration {}\n".format(only_config))
        return only_config

    if non_interactive:
        raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs))

    print("\nAvailable configurations:")
    for n, config_name in config_list:
        config_data = configs[config_name]
        expiry_date = config_data.get("expires", None)
        config_desc = config_data["description"]
        if expiry_date:
           print("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date))
        else:
           print("{}. {}\t{}".format(n, config_name, config_desc))
    print("\nPlease select one of the above configurations by its number:")
    config_n = int_input([i[0] for i in config_list])
    return config_list[config_n][1]

def choose_fragments(possibilities, parameters, non_interactive, skip_selection):
    choices = {}
    for k,v in possibilities.items():
        if skip_selection and k in skip_selection:
            print("Skipping a selection of {}, as requested on command line. The resulting bitbake configuration may require further manual adjustments.".format(k))
            continue
        choice = [o for o in v["options"] if o in parameters]
        if len(choice) > 1:
            raise Exception("Options specified on command line do not allow a single selection from possibilities {}, please remove one or more from {}".format(v["options"], parameters))
        if len(choice) == 1:
            choices[k] = choice[0]
            continue

        if non_interactive:
            raise Exception("Unable to choose from options in non-interactive mode: {}".format(v["options"]))

        print("\n" + v["description"] + ":")
        options_enumerated = list(enumerate(v["options"]))
        for n,o in options_enumerated:
            print("{}. {}".format(n, o))
        print("\nPlease select one of the above options by its number:")
        option_n = int_input([i[0] for i in options_enumerated])
        choices[k] = options_enumerated[option_n][1]
    return choices

def obtain_config(top_dir, settings, args, source_overrides, d):
    if args.config:
        config_id = args.config[0]
        config_parameters = args.config[1:]
        if os.path.exists(config_id):
            print("Reading configuration from local file\n    {}".format(config_id))
            upstream_config = {'type':'local',
                               'path':os.path.abspath(config_id),
                               'name':get_config_name(config_id),
                               'data':json.load(open(config_id))
                               }
        elif config_id.startswith("http://") or config_id.startswith("https://"):
            print("Reading configuration from network URI\n    {}".format(config_id))
            import urllib.request
            with urllib.request.urlopen(config_id) as f:
              upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)}
        else:
            print("Looking up config {} in configuration registry".format(config_id))
            registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
            registry_configs = list_registry(registry_path, with_expired=True)
            if config_id not in registry_configs:
                raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id))
            upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
        expiry_date = upstream_config['data'].get("expires", None)
        if has_expired(expiry_date):
            print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date))
    else:
        registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
        registry_configs = list_registry(registry_path, with_expired=True)
        config_id = choose_config(registry_configs, args.non_interactive)
        config_parameters = []
        upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}

    upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive)
    upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection)
    upstream_config['non-interactive-cmdline-options'] = [config_id, upstream_config['bitbake-config']['name']] + sorted(upstream_config['bitbake-config']['oe-fragment-choices'].values())
    upstream_config['source-overrides'] = source_overrides
    upstream_config['skip-selection'] = args.skip_selection
    return upstream_config

def init_config(top_dir, settings, args, d):
    stdout = sys.stdout
    def handle_task_progress(event, d):
        rate = event.rate if event.rate else ''
        progress = event.progress if event.progress > 0 else 0
        print("{}% {}                ".format(progress, rate), file=stdout, end='\r')

    create_siteconf(top_dir, args.non_interactive)
    source_overrides = json.load(open(args.source_overrides)) if args.source_overrides else {'sources':{}}
    upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
    print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))

    setupdir = os.path.join(os.path.abspath(top_dir), args.setup_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
    if os.path.exists(os.path.join(setupdir, "layers")):
        print(f"Setup already initialized in:\n    {setupdir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize in this directory, remove it, and run 'bitbake-setup init' again.")
        return

    print("Initializing a setup directory in\n    {}".format(setupdir))
    if not args.non_interactive:
        y_or_n = input('Continue? (y/N): ')
        if y_or_n != 'y':
            exit()
        print()

    os.makedirs(setupdir, exist_ok=True)

    confdir = os.path.join(setupdir, "config")
    layerdir = os.path.join(setupdir, "layers")

    os.makedirs(confdir)
    os.makedirs(layerdir)

    bb.process.run("git -C {} init -b main".format(confdir))
    # Make sure commiting doesn't fail if no default git user is configured on the machine
    bb.process.run("git -C {} config user.name bitbake-setup".format(confdir))
    bb.process.run("git -C {} config user.email bitbake-setup@not.set".format(confdir))
    bb.process.run("git -C {} commit --no-verify --allow-empty -m 'Initial commit'".format(confdir))

    bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)

    write_upstream_config(confdir, upstream_config)
    update_build(upstream_config, confdir, setupdir, layerdir, d)
    commit_config(confdir)

    bb.event.remove("bb.build.TaskProgress", None)

def get_diff(file1, file2):
    try:
        bb.process.run('diff -uNr {} {}'.format(file1, file2))
    except bb.process.ExecutionError as e:
        if e.exitcode == 1:
            return e.stdout
        else:
            raise e
    return None

def are_layers_changed(layers, layerdir, d):
    changed = False
    for r_name in layers:
        r_data = layers[r_name]
        repodir = r_data["path"]

        r_remote = r_data['git-remote']
        rev = r_remote['rev']
        branch = r_remote.get('branch', None)
        remotes = r_remote['remotes']

        for remote in remotes:
            type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
            fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
            if branch:
                fetcher = bb.fetch.FetchData("{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir), d)
            else:
                fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d)
            upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default')
            rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir)))
            local_revision = rev_parse_result[0].strip()
            if upstream_revision != local_revision:
                changed = True
                print('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remotes[remote]["uri"], os.path.join(layerdir, repodir), rev, local_revision, upstream_revision))

    return changed

def build_status(top_dir, settings, args, d, update=False):
    setupdir = args.setup_dir

    confdir = os.path.join(setupdir, "config")
    layerdir = os.path.join(setupdir, "layers")

    current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))

    args.config = current_upstream_config['non-interactive-cmdline-options']
    args.non_interactive = True
    args.skip_selection = current_upstream_config['skip-selection']
    source_overrides = current_upstream_config["source-overrides"]
    new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)

    write_upstream_config(confdir, new_upstream_config)
    config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]

    if config_diff:
        print('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff))
        if update:
            commit_config(confdir)
            update_build(new_upstream_config, confdir, setupdir, layerdir, d)
        else:
            bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
        return

    if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
        if update:
            update_build(current_upstream_config, confdir, setupdir, layerdir, d)
        return

    print("\nConfiguration in {} has not changed.".format(setupdir))

def build_update(top_dir, settings, args, d):
    build_status(top_dir, settings, args, d, update=True)

def do_fetch(fetcher, dir):
    # git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch
    # and we need to set up smth similar here
    fetchlogdir = os.path.join(dir, 'logs')
    os.makedirs(fetchlogdir, exist_ok=True)
    fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
    with open(fetchlog, 'a') as f:
        oldstdout = sys.stdout
        sys.stdout = f
        fetcher.download()
        fetcher.unpack(dir)
        sys.stdout = oldstdout

def update_registry(registry, cachedir, d):
    registrydir = 'configurations'
    if registry.startswith("."):
        full_registrydir = os.path.join(os.getcwd(), registry, registrydir)
    elif registry.startswith("/"):
        full_registrydir = os.path.join(registry, registrydir)
    else:
        full_registrydir = os.path.join(cachedir, registrydir)
        print("Fetching configuration registry\n    {}\ninto\n    {}".format(registry, full_registrydir))
        fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d)
        do_fetch(fetcher, cachedir)
    return full_registrydir

def has_expired(expiry_date):
    if expiry_date:
        return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date)
    return False

def list_registry(registry_path, with_expired):
    json_data = {}

    for root, dirs, files in os.walk(registry_path):
        for f in files:
            if f.endswith('.conf.json'):
                config_name = get_config_name(f)
                config_data = json.load(open(os.path.join(root, f)))
                config_desc = config_data["description"]
                expiry_date = config_data.get("expires", None)
                if expiry_date:
                    if with_expired or not has_expired(expiry_date):
                        json_data[config_name] = {"description": config_desc, "expires": expiry_date}
                else:
                    json_data[config_name] = {"description": config_desc}
    return json_data

def list_configs(top_dir, settings, args, d):
    registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
    json_data = list_registry(registry_path, args.with_expired)
    print("\nAvailable configurations:")
    for config_name, config_data in json_data.items():
        expiry_date = config_data.get("expires", None)
        config_desc = config_data["description"]
        if expiry_date:
           if args.with_expired or not has_expired(expiry_date):
               print("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date))
        else:
           print("{}\t{}".format(config_name, config_desc))
    print("\nRun 'init' with one of the above configuration identifiers to set up a build.")

    if args.write_json:
        with open(args.write_json, 'w') as f:
            json.dump(json_data, f, sort_keys=True, indent=4)
        print("Available configurations written into {}".format(args.write_json))

def install_buildtools(top_dir, settings, args, d):
    buildtools_install_dir = os.path.join(args.setup_dir, 'buildtools')
    if os.path.exists(buildtools_install_dir):
        if not args.force:
            print("Buildtools are already installed in {}.".format(buildtools_install_dir))
            env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*'))
            if env_scripts:
                print("If you wish to use them, you need to source the environment setup script e.g.")
                for s in env_scripts:
                    print("$ . {}".format(s))
            print("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.")
            return
        shutil.rmtree(buildtools_install_dir)

    install_buildtools = os.path.join(args.setup_dir, 'layers/oe-scripts/install-buildtools')
    buildtools_download_dir = os.path.join(args.setup_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
    print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
    subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)

def create_siteconf(top_dir, non_interactive=True):
    siteconfpath = os.path.join(top_dir, 'site.conf')
    print('A common site.conf file will be created, please edit or replace before running builds\n    {}\n'.format(siteconfpath))
    if not non_interactive:
        y_or_n = input('Proceed? (y/N): ')
        if y_or_n != 'y':
            exit()

    os.makedirs(os.path.dirname(top_dir), exist_ok=True)
    if os.path.exists(siteconfpath):
        backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
        os.rename(siteconfpath, backup_siteconf)
        print("Previous settings are in {}".format(backup_siteconf))
    with open(siteconfpath, 'w') as siteconffile:
        siteconffile.write('# This file is intended for build host-specific bitbake settings\n')

def topdir_settings_path(top_dir):
    return os.path.join(top_dir, 'settings.conf')

def global_settings_path(args):
    return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf')

def load_settings(settings_path):
    settings = configparser.ConfigParser()
    if os.path.exists(settings_path):
        print('Loading settings from\n    {}\n'.format(settings_path))
        settings.read_file(open(settings_path))
    return settings

def change_setting(top_dir, args):
    if vars(args)['global']:
        settings_path = global_settings_path(args)
    else:
        settings_path = topdir_settings_path(top_dir)
    settings = load_settings(settings_path)

    if args.subcommand == 'set':
        if args.section not in settings.keys():
            settings[args.section] = {}
        settings[args.section][args.setting] = args.value
        print(f"From section '{args.section}' the setting '{args.setting}' was changed to '{args.value}'")
    if args.subcommand == 'unset':
        if args.section in settings.keys() and args.setting in settings[args.section].keys():
            del settings[args.section][args.setting]
            print(f"From section '{args.section}' the setting '{args.setting}' has been removed")

    os.makedirs(os.path.dirname(settings_path), exist_ok=True)
    with open(settings_path, 'w') as settingsfile:
        settings.write(settingsfile)
    print(f"Settings written to {settings_path}")

def list_settings(all_settings):
    for section, section_settings in all_settings.items():
        for key, value in section_settings.items():
            print("{} {} {}".format(section, key, value))

def settings_func(top_dir, all_settings, args):
    if args.subcommand == 'list':
        list_settings(all_settings)
    elif args.subcommand == 'set' or args.subcommand == 'unset':
        change_setting(top_dir, args)

def get_setup_dir_via_bbpath():
    bbpath = os.environ.get('BBPATH')
    if bbpath:
        bitbake_dir = os.path.normpath(bbpath.split(':')[0])
        if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
            setup_dir = os.path.dirname(bitbake_dir)
            return setup_dir
    return None

def get_top_dir(args, settings):
    setup_dir_via_bbpath = get_setup_dir_via_bbpath()
    if setup_dir_via_bbpath:
        top_dir = os.path.dirname(setup_dir_via_bbpath)
        if os.path.exists(topdir_settings_path(top_dir)):
            return top_dir

    if hasattr(args, 'setup_dir'):
        top_dir = os.path.dirname(os.path.normpath(args.setup_dir))
        return top_dir

    top_dir_prefix = settings['default']['top-dir-prefix']
    top_dir_name = settings['default']['top-dir-name']
    return os.path.join(top_dir_prefix, top_dir_name)

def merge_settings(builtin_settings, global_settings, topdir_settings, cmdline_settings):
    all_settings = builtin_settings

    for s in (global_settings, topdir_settings):
        for section, section_settings in s.items():
            for setting, value in section_settings.items():
                if section not in all_settings.keys():
                    all_settings[section] = {}
                all_settings[section][setting] = value

    for (section, setting, value) in cmdline_settings:
        if section not in all_settings.keys():
            all_settings[section] = {}
        all_settings[section][setting] = value

    return all_settings

def main():
    def add_setup_dir_arg(parser):
        setup_dir = get_setup_dir_via_bbpath()
        if setup_dir:
            parser.add_argument('--setup-dir', default=setup_dir, help="Path to the setup, default is %(default)s via BBPATH")
        else:
            parser.add_argument('--setup-dir', required=True, help="Path to the setup")

    parser = argparse.ArgumentParser(
        description="BitBake setup utility. Run with 'init' argument to get started.",
        epilog="Use %(prog)s <subcommand> --help to get help on a specific command"
        )
    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
    parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
    parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.')
    parser.add_argument('--global-settings', action='store', metavar='PATH', help='Path to the global settings file.')
    parser.add_argument('--setting', default=[], action='append', dest='cmdline_settings',
                        nargs=3, metavar=('SECTION', 'SETTING', 'VALUE'),
                        help='Modify a setting (for this bitbake-setup invocation only), for example "--setting default top-dir-prefix /path/to/top/dir".')

    subparsers = parser.add_subparsers()

    parser_list = subparsers.add_parser('list', help='List available configurations')
    parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.')
    parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
    parser_list.set_defaults(func=list_configs)

    parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a setup from it')
    parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
    parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
    parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
    parser_init.add_argument('--setup-dir-name', action='store', help='A custom setup directory name under the top directory.')
    parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
    parser_init.set_defaults(func=init_config)

    parser_status = subparsers.add_parser('status', help='Check if the setup needs to be synchronized with configuration')
    add_setup_dir_arg(parser_status)
    parser_status.set_defaults(func=build_status)

    parser_update = subparsers.add_parser('update', help='Update a setup to be in sync with configuration')
    add_setup_dir_arg(parser_update)
    parser_update.set_defaults(func=build_update)

    parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
    add_setup_dir_arg(parser_install_buildtools)
    parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
    parser_install_buildtools.set_defaults(func=install_buildtools)

    parser_settings_arg_global = argparse.ArgumentParser(add_help=False)
    parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory")

    parser_settings = subparsers.add_parser('settings',
                                            help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of configuration registry, downloads directory and other settings specific to a top directory)')
    parser_settings.set_defaults(func=settings_func)

    subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file")

    parser_settings_list = subparser_settings.add_parser('list',
                                                         help="List all settings with their values")

    parser_settings_set = subparser_settings.add_parser('set', parents=[parser_settings_arg_global],
                                                        help="In a Section, set a setting to a certain value")
    parser_settings_set.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'")
    parser_settings_set.add_argument("setting", metavar="<setting>", help="Name of a setting")
    parser_settings_set.add_argument("value", metavar="<value>", help="The setting value")

    parser_settings_unset = subparser_settings.add_parser('unset', parents=[parser_settings_arg_global],
                                                          help="Unset a setting, e.g. 'bitbake-setup settings unset default registry' would revert to the registry setting in a global settings file")
    parser_settings_unset.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'")
    parser_settings_unset.add_argument("setting", metavar="<setting>", help="The setting to remove")

    args = parser.parse_args()

    logging.basicConfig(stream=sys.stdout)
    if args.debug:
        logger.setLevel(logging.DEBUG)
    elif args.quiet:
        logger.setLevel(logging.ERROR)

    # Need to re-run logger_create with color argument
    # (will be the same logger since it has the same name)
    bb.msg.logger_create('bitbake-setup', output=sys.stdout,
                         color=args.color,
                         level=logger.getEffectiveLevel())

    if 'func' in args:
        if hasattr(args, 'setup_dir'):
            if not os.path.exists(os.path.join(args.setup_dir,'build', 'init-build-env')):
                print("Not a valid setup directory: build/init-build-env does not exist in {}".format(args.setup_dir))
                return

        if not hasattr(args, 'non_interactive'):
            args.non_interactive = True

        builtin_settings = {}
        builtin_settings['default'] = {
                         'top-dir-prefix':os.path.expanduser('~'),
                         'top-dir-name':'bitbake-builds',
                         'registry':default_registry,
                         }

        global_settings = load_settings(global_settings_path(args))
        top_dir = get_top_dir(args, merge_settings(builtin_settings, global_settings, {}, args.cmdline_settings))

        # This cannot be set with the rest of the builtin settings as top_dir needs to be determined first
        builtin_settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads')

        topdir_settings = load_settings(topdir_settings_path(top_dir))
        all_settings = merge_settings(builtin_settings, global_settings, topdir_settings, args.cmdline_settings)

        if args.func == settings_func:
            settings_func(top_dir, all_settings, args)
            return

        print('Bitbake-setup is using {} as top directory ("bitbake-setup settings --help" shows how to change it).\n'.format(top_dir, global_settings_path(args)))

        d = init_bb_cache(top_dir, all_settings, args)
        args.func(top_dir, all_settings, args, d)
        save_bb_cache()
    else:
        from argparse import Namespace
        parser.print_help()

main()