poky/bitbake/bin/bitbake-setup

#!/usr/bin/env python3

#
# SPDX-License-Identifier: GPL-2.0-only
#

import logging
import os
import sys
import argparse
import warnings
import json
import shutil
import time
import stat
import tempfile
import configparser
import datetime
import glob
import subprocess

default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")

bindir = os.path.abspath(os.path.dirname(__file__))
sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')]

import bb.msg
import bb.process

logger = bb.msg.logger_create('bitbake-setup', sys.stdout)

def cache_dir(top_dir):
    return os.path.join(top_dir, '.bitbake-setup-cache')

def init_bb_cache(top_dir, settings, args):
    dldir = settings["default"]["dl-dir"]
    bb_cachedir = os.path.join(cache_dir(top_dir), 'bitbake-cache')

    d = bb.data.init()
    d.setVar("DL_DIR", dldir)
    d.setVar("BB_CACHEDIR", bb_cachedir)
    d.setVar("__BBSRCREV_SEEN", "1")
    if args.no_network:
        d.setVar("BB_SRCREV_POLICY", "cache")
    bb.fetch.fetcher_init(d)
    return d

def save_bb_cache():
    bb.fetch2.fetcher_parse_save()
    bb.fetch2.fetcher_parse_done()

def get_config_name(config):
    suffix = '.conf.json'
    config_file = os.path.basename(config)
    if config_file.endswith(suffix):
        return config_file[:-len(suffix)]
    else:
        raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix))

def write_config(config, config_dir):
    with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
        json.dump(config, s, sort_keys=True, indent=4)

def commit_config(config_dir):
    bb.process.run("git -C {} add .".format(config_dir))
    bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime()))

def _write_layer_list(dest, repodirs):
    layers = []
    for r in repodirs:
        for root, dirs, files in os.walk(os.path.join(dest,r)):
            if os.path.basename(root) == 'conf' and 'layer.conf' in files:
                layers.append(os.path.relpath(os.path.dirname(root), dest))
    layers_f = os.path.join(dest, ".oe-layers.json")
    with open(layers_f, 'w') as f:
        json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)

def checkout_layers(layers, layerdir, d):
    repodirs = []
    oesetupbuild = None
    print("Fetching layer/tool repositories into {}".format(layerdir))
    for r_name in layers:
        r_data = layers[r_name]
        repodir = r_data["path"]
        repodirs.append(repodir)

        r_remote = r_data['git-remote']
        rev = r_remote['rev']
        branch = r_remote.get('branch', None)
        remotes = r_remote['remotes']

        for remote in remotes:
            type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
            fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
            print("    {}".format(r_name))
            if branch:
                fetcher = bb.fetch.Fetch(["{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir)], d)
            else:
                fetcher = bb.fetch.Fetch(["{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir)], d)
            do_fetch(fetcher, layerdir)

        if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
            oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
            oeinitbuildenvdir = os.path.join(layerdir, repodir)

    print("        ")
    _write_layer_list(layerdir, repodirs)

    if oesetupbuild:
        links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'oe-init-build-env-dir': oeinitbuildenvdir}
        for l,t in links.items():
            symlink = os.path.join(layerdir, l)
            if os.path.lexists(symlink):
                os.remove(symlink)
            os.symlink(os.path.relpath(t,layerdir),symlink)

def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
    def _setup_build_conf(layers, build_conf_dir):
        os.makedirs(build_conf_dir)
        layers_s = []
        for l in layers:
            if l.startswith("{THISDIR}/"):
                if thisdir:
                    l = l.format(THISDIR=thisdir)
                else:
                    raise Exception("Configuration is using {THISDIR} to specify " \
                    "a layer path relative to itself. This can be done only " \
                    "when the configuration is specified by its path on local " \
                    "disk, not when it's in a registry or is fetched over http.")
            if not os.path.isabs(l):
                l = os.path.join(layerdir, l)
            layers_s.append("  {} \\".format(l))
        layers_s = "\n".join(layers_s)
        bblayers_conf = """BBLAYERS ?= " \\
{}
  "
""".format(layers_s)
        with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f:
            f.write(bblayers_conf)

        local_conf = """#
# This file is intended for local configuration tweaks.
#
# If you would like to publish and share changes made to this file,
# it is recommended to put them into a distro config, or to create
# layer fragments from changes made here.
#
"""
        with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f:
            f.write(local_conf)

        with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f:
            f.write("")

        with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f:
            f.write(bitbake_config["description"] + "\n")

        with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f:
            f.write("")

    def _make_init_build_env(builddir, oeinitbuildenvdir):
        builddir = os.path.realpath(builddir)
        cmd = "cd {}\nset {}\n. ./oe-init-build-env\n".format(oeinitbuildenvdir, builddir)
        initbuild_in_builddir = os.path.join(builddir, 'init-build-env')

        with open(initbuild_in_builddir, 'w') as f:
            f.write("# init-build-env wrapper created by bitbake-setup\n")
            f.write(cmd + '\n')

    def _prepend_passthrough_to_init_build_env(builddir):
        env = bitbake_config.get("bb-env-passthrough-additions")
        if not env:
            return

        initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
        with open(initbuild_in_builddir) as f:
            content = f.read()

        joined = " \\\n".join(env)
        env = "export BB_ENV_PASSTHROUGH_ADDITIONS=\" \\\n"
        env += "${BB_ENV_PASSTHROUGH_ADDITIONS} \\\n"
        env += joined
        env += '"'

        with open(initbuild_in_builddir, 'w') as f:
            f.write("# environment passthrough added by bitbake-setup\n")
            f.write(env + '\n')
            f.write('\n')
            f.write(content)

    bitbake_builddir = os.path.join(builddir, "build")
    print("Setting up bitbake configuration in\n    {}\n".format(bitbake_builddir))

    template = bitbake_config.get("oe-template")
    layers = bitbake_config.get("bb-layers")
    if not template and not layers:
        print("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.")
        return
    oesetupbuild = os.path.join(layerdir, 'setup-build')
    if template and not os.path.exists(oesetupbuild):
        raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template))

    bitbake_confdir = os.path.join(bitbake_builddir, 'conf')
    backup_bitbake_confdir = bitbake_confdir + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
    if os.path.exists(bitbake_confdir):
        os.rename(bitbake_confdir, backup_bitbake_confdir)

    if layers:
        _setup_build_conf(layers, bitbake_confdir)

    if template:
        bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir))
    else:
        oeinitbuildenvdir = os.path.join(layerdir, 'oe-init-build-env-dir')
        if not os.path.exists(os.path.join(oeinitbuildenvdir, "oe-init-build-env")):
            print("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment")
            return
        _make_init_build_env(bitbake_builddir, os.path.realpath(oeinitbuildenvdir))

    _prepend_passthrough_to_init_build_env(bitbake_builddir)

    siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
    siteconf = os.path.normpath(os.path.join(builddir, '..', "site.conf"))
    if os.path.lexists(siteconf_symlink):
        os.remove(symlink)
    os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)


    init_script = os.path.join(bitbake_builddir, "init-build-env")
    shell = "bash"
    fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values())
    if fragments:
        bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments)))

    if os.path.exists(backup_bitbake_confdir):
        bitbake_config_diff = get_diff(backup_bitbake_confdir, bitbake_confdir)
        if bitbake_config_diff:
            print("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir))
            print("The bitbake configuration has changed:")
            print(bitbake_config_diff)
        else:
            shutil.rmtree(backup_bitbake_confdir)

    print("This bitbake configuration provides:\n    {}\n".format(bitbake_config["description"]))

    readme = """{}\n\nAdditional information is in {} and {}\n
Source the environment using '. {}' to run builds from the command line.
The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf
""".format(
        bitbake_config["description"],
        os.path.join(bitbake_builddir,'conf/conf-summary.txt'),
        os.path.join(bitbake_builddir,'conf/conf-notes.txt'),
        init_script,
        bitbake_builddir
        )
    readme_file = os.path.join(bitbake_builddir, "README")
    with open(readme_file, 'w') as f:
        f.write(readme)
    print("Usage instructions and additional information are in\n     {}\n".format(readme_file))
    print("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n    {}/conf\n".format(bitbake_builddir))
    print("To run builds, source the environment using\n    . {}".format(init_script))

def get_registry_config(registry_path, id):
    for root, dirs, files in os.walk(registry_path):
        for f in files:
            if f.endswith('.conf.json') and id == get_config_name(f):
                return os.path.join(root, f)
    raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))

def update_build(config, confdir, builddir, layerdir, d):
    layer_config = config["data"]["sources"]
    layer_overrides = config["source-overrides"]["sources"]
    for k,v in layer_overrides.items():
        if k in layer_config:
            layer_config[k]["git-remote"] = v["git-remote"]
    checkout_layers(layer_config, layerdir, d)
    bitbake_config = config["bitbake-config"]
    thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
    setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir)

def int_input(allowed_values):
    n = None
    while n is None:
        try:
            n = int(input())
        except ValueError:
            print('Not a valid number, please try again:')
            continue
        if n not in allowed_values:
            print('Number {} not one of {}, please try again:'.format(n, allowed_values))
            n = None
    return n

def flatten_bitbake_configs(configs):
    def merge_configs(c1,c2):
        c_merged = {}
        for k,v in c2.items():
            if k not in c1.keys():
                c_merged[k] = v
        for k,v in c1.items():
            if k not in c2.keys():
                c_merged[k] = v
            else:
               c_merged[k] = c1[k] + c2[k]
        del c_merged['configurations']
        return c_merged

    flattened_configs = []
    for c in configs:
        if 'configurations' not in c:
            flattened_configs.append(c)
        else:
            for sub_c in flatten_bitbake_configs(c['configurations']):
                flattened_configs.append(merge_configs(c, sub_c))
    return flattened_configs

def choose_bitbake_config(configs, parameters, non_interactive):
    flattened_configs = flatten_bitbake_configs(configs)
    configs_dict = {i["name"]:i for i in flattened_configs}

    if parameters:
        config_id = parameters[0]
        if config_id not in configs_dict:
            raise Exception("Bitbake configuration {} not found; replace with one of {}".format(config_id, configs_dict))
        return configs_dict[config_id]

    enumerated_configs = list(enumerate(flattened_configs))
    if len(enumerated_configs) == 1:
        only_config = flattened_configs[0]
        print("\nSelecting the only available bitbake configuration {}".format(only_config["name"]))
        return only_config

    if non_interactive:
        raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict))

    print("\nAvailable bitbake configurations:")
    for n, config_data in enumerated_configs:
        print("{}. {}\t{}".format(n, config_data["name"], config_data["description"]))
    print("\nPlease select one of the above bitbake configurations by its number:")
    config_n = int_input([i[0] for i in enumerated_configs])
    return flattened_configs[config_n]

def choose_config(configs, non_interactive):
    not_expired_configs = [k for k in configs.keys() if not has_expired(configs[k].get("expires", None))]
    config_list = list(enumerate(not_expired_configs))
    if len(config_list) == 1:
        only_config = config_list[0][1]
        print("\nSelecting the only available configuration {}\n".format(only_config))
        return only_config

    if non_interactive:
        raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs))

    print("\nAvailable configurations:")
    for n, config_name in config_list:
        config_data = configs[config_name]
        expiry_date = config_data.get("expires", None)
        config_desc = config_data["description"]
        if expiry_date:
           print("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date))
        else:
           print("{}. {}\t{}".format(n, config_name, config_desc))
    print("\nPlease select one of the above configurations by its number:")
    config_n = int_input([i[0] for i in config_list])
    return config_list[config_n][1]

def choose_fragments(possibilities, parameters, non_interactive, skip_selection):
    choices = {}
    for k,v in possibilities.items():
        if skip_selection and k in skip_selection:
            print("Skipping a selection of {}, as requested on command line. The resulting bitbake configuration may require further manual adjustments.".format(k))
            continue
        choice = [o for o in v["options"] if o in parameters]
        if len(choice) > 1:
            raise Exception("Options specified on command line do not allow a single selection from possibilities {}, please remove one or more from {}".format(v["options"], parameters))
        if len(choice) == 1:
            choices[k] = choice[0]
            continue

        if non_interactive:
            raise Exception("Unable to choose from options in non-interactive mode: {}".format(v["options"]))

        print("\n" + v["description"] + ":")
        options_enumerated = list(enumerate(v["options"]))
        for n,o in options_enumerated:
            print("{}. {}".format(n, o))
        print("\nPlease select one of the above options by its number:")
        option_n = int_input([i[0] for i in options_enumerated])
        choices[k] = options_enumerated[option_n][1]
    return choices

def obtain_config(top_dir, settings, args, source_overrides, d):
    if args.config:
        config_id = args.config[0]
        config_parameters = args.config[1:]
        if os.path.exists(config_id):
            print("Reading configuration from local file\n    {}".format(config_id))
            upstream_config = {'type':'local',
                               'path':os.path.abspath(config_id),
                               'name':get_config_name(config_id),
                               'data':json.load(open(config_id))
                               }
        elif config_id.startswith("http://") or config_id.startswith("https://"):
            print("Reading configuration from network URI\n    {}".format(config_id))
            import urllib.request
            with urllib.request.urlopen(config_id) as f:
              upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)}
        else:
            print("Looking up config {} in configuration registry".format(config_id))
            registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
            registry_configs = list_registry(registry_path, with_expired=True)
            if config_id not in registry_configs:
                raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id))
            upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
        expiry_date = upstream_config['data'].get("expires", None)
        if has_expired(expiry_date):
            print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date))
    else:
        registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
        registry_configs = list_registry(registry_path, with_expired=True)
        config_id = choose_config(registry_configs, args.non_interactive)
        config_parameters = []
        upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}

    upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive)
    upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection)
    upstream_config['non-interactive-cmdline-options'] = [config_id, upstream_config['bitbake-config']['name']] + sorted(upstream_config['bitbake-config']['oe-fragment-choices'].values())
    upstream_config['source-overrides'] = source_overrides
    upstream_config['skip-selection'] = args.skip_selection
    return upstream_config

def init_config(top_dir, settings, args, d):
    stdout = sys.stdout
    def handle_task_progress(event, d):
        rate = event.rate if event.rate else ''
        progress = event.progress if event.progress > 0 else 0
        print("{}% {}                ".format(progress, rate), file=stdout, end='\r')

    create_siteconf(top_dir, args.non_interactive)
    source_overrides = json.load(open(args.source_overrides)) if args.source_overrides else {'sources':{}}
    upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
    print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))

    builddir = os.path.join(os.path.abspath(top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
    if os.path.exists(os.path.join(builddir, "layers")):
        print(f"Build already initialized in:\n    {builddir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize a build in this directory, remove it, and run 'bitbake-setup init' again.")
        return

    print("Initializing a build in\n    {}".format(builddir))
    if not args.non_interactive:
        y_or_n = input('Continue? (y/N): ')
        if y_or_n != 'y':
            exit()
        print()

    os.makedirs(builddir, exist_ok=True)

    confdir = os.path.join(builddir, "config")
    layerdir = os.path.join(builddir, "layers")

    os.makedirs(confdir)
    os.makedirs(layerdir)

    bb.process.run("git -C {} init -b main".format(confdir))
    # Make sure commiting doesn't fail if no default git user is configured on the machine
    bb.process.run("git -C {} config user.name bitbake-setup".format(confdir))
    bb.process.run("git -C {} config user.email bitbake-setup@not.set".format(confdir))
    bb.process.run("git -C {} commit --no-verify --allow-empty -m 'Initial commit'".format(confdir))

    bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)

    write_config(upstream_config, confdir)
    commit_config(confdir)
    update_build(upstream_config, confdir, builddir, layerdir, d)

    bb.event.remove("bb.build.TaskProgress", None)

def get_diff(file1, file2):
    try:
        bb.process.run('diff -uNr {} {}'.format(file1, file2))
    except bb.process.ExecutionError as e:
        if e.exitcode == 1:
            return e.stdout
        else:
            raise e
    return None

def are_layers_changed(layers, layerdir, d):
    changed = False
    for r_name in layers:
        r_data = layers[r_name]
        repodir = r_data["path"]

        r_remote = r_data['git-remote']
        rev = r_remote['rev']
        branch = r_remote.get('branch', None)
        remotes = r_remote['remotes']

        for remote in remotes:
            type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
            fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
            if branch:
                fetcher = bb.fetch.FetchData("{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir), d)
            else:
                fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d)
            upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default')
            rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir)))
            local_revision = rev_parse_result[0].strip()
            if upstream_revision != local_revision:
                changed = True
                print('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remotes[remote]["uri"], os.path.join(layerdir, repodir), rev, local_revision, upstream_revision))

    return changed

def build_status(top_dir, settings, args, d, update=False):
    builddir = args.build_dir

    confdir = os.path.join(builddir, "config")
    layerdir = os.path.join(builddir, "layers")

    current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))

    args.config = current_upstream_config['non-interactive-cmdline-options']
    args.non_interactive = True
    args.skip_selection = current_upstream_config['skip-selection']
    source_overrides = current_upstream_config["source-overrides"]
    new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)

    write_config(new_upstream_config, confdir)
    config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]

    if config_diff:
        print('\nConfiguration in {} has changed:\n{}'.format(builddir, config_diff))
        if update:
            commit_config(confdir)
            update_build(new_upstream_config, confdir, builddir, layerdir, d)
        else:
            bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
        return

    if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
        if update:
            update_build(current_upstream_config, confdir, builddir, layerdir, d)
        return

    print("\nConfiguration in {} has not changed.".format(builddir))

def build_update(top_dir, settings, args, d):
    build_status(top_dir, settings, args, d, update=True)

def do_fetch(fetcher, dir):
    # git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch
    # and we need to set up smth similar here
    fetchlogdir = os.path.join(dir, 'logs')
    os.makedirs(fetchlogdir, exist_ok=True)
    fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
    with open(fetchlog, 'a') as f:
        oldstdout = sys.stdout
        sys.stdout = f
        fetcher.download()
        fetcher.unpack(dir)
        sys.stdout = oldstdout

def update_registry(registry, cachedir, d):
    registrydir = 'configurations'
    if registry.startswith("."):
        full_registrydir = os.path.join(os.getcwd(), registry, registrydir)
    elif registry.startswith("/"):
        full_registrydir = os.path.join(registry, registrydir)
    else:
        full_registrydir = os.path.join(cachedir, registrydir)
        print("Fetching configuration registry\n    {}\ninto\n    {}".format(registry, full_registrydir))
        fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d)
        do_fetch(fetcher, cachedir)
    return full_registrydir

def has_expired(expiry_date):
    if expiry_date:
        return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date)
    return False

def list_registry(registry_path, with_expired):
    json_data = {}

    for root, dirs, files in os.walk(registry_path):
        for f in files:
            if f.endswith('.conf.json'):
                config_name = get_config_name(f)
                config_data = json.load(open(os.path.join(root, f)))
                config_desc = config_data["description"]
                expiry_date = config_data.get("expires", None)
                if expiry_date:
                    if with_expired or not has_expired(expiry_date):
                        json_data[config_name] = {"description": config_desc, "expires": expiry_date}
                else:
                    json_data[config_name] = {"description": config_desc}
    return json_data

def list_configs(top_dir, settings, args, d):
    registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
    json_data = list_registry(registry_path, args.with_expired)
    print("\nAvailable configurations:")
    for config_name, config_data in json_data.items():
        expiry_date = config_data.get("expires", None)
        config_desc = config_data["description"]
        if expiry_date:
           if args.with_expired or not has_expired(expiry_date):
               print("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date))
        else:
           print("{}\t{}".format(config_name, config_desc))
    print("\nRun 'init' with one of the above configuration identifiers to set up a build.")

    if args.write_json:
        with open(args.write_json, 'w') as f:
            json.dump(json_data, f, sort_keys=True, indent=4)
        print("Available configurations written into {}".format(args.write_json))

def install_buildtools(top_dir, settings, args, d):
    buildtools_install_dir = os.path.join(args.build_dir, 'buildtools')
    if os.path.exists(buildtools_install_dir):
        if not args.force:
            print("Buildtools are already installed in {}.".format(buildtools_install_dir))
            env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*'))
            if env_scripts:
                print("If you wish to use them, you need to source the environment setup script e.g.")
                for s in env_scripts:
                    print("$ . {}".format(s))
            print("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.")
            return
        shutil.rmtree(buildtools_install_dir)

    install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools')
    buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
    print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
    subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)

def default_settings_path(top_dir):
    return os.path.join(top_dir, 'settings.conf')

def create_siteconf(top_dir, non_interactive=True):
    siteconfpath = os.path.join(top_dir, 'site.conf')
    print('A common site.conf file will be created, please edit or replace before running builds\n    {}\n'.format(siteconfpath))
    if not non_interactive:
        y_or_n = input('Proceed? (y/N): ')
        if y_or_n != 'y':
            exit()

    os.makedirs(os.path.dirname(top_dir), exist_ok=True)
    if os.path.exists(siteconfpath):
        backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
        os.rename(siteconfpath, backup_siteconf)
        print("Previous settings are in {}".format(backup_siteconf))
    with open(siteconfpath, 'w') as siteconffile:
        siteconffile.write('# This file is intended for build host-specific bitbake settings\n')

def global_settings_path(args):
    return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf')

def load_settings(settings_path):
    settings = configparser.ConfigParser()
    if os.path.exists(settings_path):
        print('Loading settings from\n    {}\n'.format(settings_path))
        settings.read_file(open(settings_path))
    return settings

def change_setting(top_dir, args):
    if vars(args)['global']:
        settings_path = global_settings_path(args)
    else:
        settings_path = default_settings_path(top_dir)
    settings = load_settings(settings_path)

    if args.subcommand == 'set':
        if args.section not in settings.keys():
            settings[args.section] = {}
        settings[args.section][args.setting] = args.value
        print(f"From section '{args.section}' the setting '{args.setting}' was changed to '{args.value}'")
    if args.subcommand == 'unset':
        if args.section in settings.keys() and args.setting in settings[args.section].keys():
            del settings[args.section][args.setting]
            print(f"From section '{args.section}' the setting '{args.setting}' has been removed")

    os.makedirs(os.path.dirname(settings_path), exist_ok=True)
    with open(settings_path, 'w') as settingsfile:
        settings.write(settingsfile)
    print(f"Settings written to {settings_path}")

def list_settings(all_settings):
    for section, section_settings in all_settings.items():
        for key, value in section_settings.items():
            print("{} {} {}".format(section, key, value))

def settings_func(top_dir, all_settings, args):
    if args.subcommand == 'list':
        list_settings(all_settings)
    elif args.subcommand == 'set' or args.subcommand == 'unset':
        change_setting(top_dir, args)

def get_build_dir_via_bbpath():
    bbpath = os.environ.get('BBPATH')
    if bbpath:
        bitbake_dir = os.path.normpath(bbpath.split(':')[0])
        if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
            build_dir = os.path.dirname(bitbake_dir)
            return build_dir
    return None

def get_top_dir(args, settings):
    build_dir_via_bbpath = get_build_dir_via_bbpath()
    if build_dir_via_bbpath:
        top_dir = os.path.dirname(build_dir_via_bbpath)
        if os.path.exists(default_settings_path(top_dir)):
            return top_dir

    if hasattr(args, 'build_dir'):
        top_dir = os.path.dirname(os.path.normpath(args.build_dir))
        return top_dir

    top_dir_prefix = settings['default']['top-dir-prefix']
    top_dir_name = settings['default']['top-dir-name']
    return os.path.join(top_dir_prefix, top_dir_name)

def merge_settings(builtin_settings, global_settings, local_settings, cmdline_settings):
    all_settings = builtin_settings

    for s in (global_settings, local_settings):
        for section, section_settings in s.items():
            for setting, value in section_settings.items():
                all_settings[section][setting] = value

    for (section, setting, value) in cmdline_settings:
        all_settings[section][setting] = value

    return all_settings

def main():
    def add_build_dir_arg(parser):
        build_dir = get_build_dir_via_bbpath()
        if build_dir:
            parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH")
        else:
            parser.add_argument('--build-dir', required=True, help="Path to the build")

    parser = argparse.ArgumentParser(
        description="BitBake setup utility. Run with 'init' argument to get started.",
        epilog="Use %(prog)s <subcommand> --help to get help on a specific command"
        )
    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
    parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
    parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.')
    parser.add_argument('--global-settings', action='store', metavar='PATH', help='Path to the global settings file.')
    parser.add_argument('--setting', default=[], action='append', dest='cmdline_settings',
                        nargs=3, metavar=('SECTION', 'SETTING', 'VALUE'),
                        help='Modify a setting (for this bitbake-setup invocation only), for example "--setting default top-dir-prefix /path/to/top/dir".')

    subparsers = parser.add_subparsers()

    parser_list = subparsers.add_parser('list', help='List available configurations')
    parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.')
    parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
    parser_list.set_defaults(func=list_configs)

    parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it')
    parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
    parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
    parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
    parser_init.add_argument('--build-dir-name', action='store', help='A custom build directory name under the top directory.')
    parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
    parser_init.set_defaults(func=init_config)

    parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration')
    add_build_dir_arg(parser_status)
    parser_status.set_defaults(func=build_status)

    parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration')
    add_build_dir_arg(parser_update)
    parser_update.set_defaults(func=build_update)

    parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
    add_build_dir_arg(parser_install_buildtools)
    parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
    parser_install_buildtools.set_defaults(func=install_buildtools)

    parser_settings_arg_global = argparse.ArgumentParser(add_help=False)
    parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory")

    parser_settings = subparsers.add_parser('settings', parents=[parser_settings_arg_global],
                                            help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of build configuration registry, downloads directory and other settings specific to a top directory)')
    parser_settings.set_defaults(func=settings_func)

    subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file")

    parser_settings_list = subparser_settings.add_parser('list',
                                                         help="List all settings with their values")

    parser_settings_set = subparser_settings.add_parser('set', parents=[parser_settings_arg_global],
                                                        help="In a Section, set a setting to a certain value")
    parser_settings_set.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'")
    parser_settings_set.add_argument("setting", metavar="<setting>", help="Name of a setting")
    parser_settings_set.add_argument("value", metavar="<value>", help="The setting value")

    parser_settings_unset = subparser_settings.add_parser('unset', parents=[parser_settings_arg_global],
                                                          help="Unset a setting, e.g. 'bitbake-setup settings unset default registry' would revert to the registry setting in a global settings file")
    parser_settings_unset.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'")
    parser_settings_unset.add_argument("setting", metavar="<setting>", help="The setting to remove")

    args = parser.parse_args()

    logging.basicConfig(stream=sys.stdout)
    if args.debug:
        logger.setLevel(logging.DEBUG)
    elif args.quiet:
        logger.setLevel(logging.ERROR)

    # Need to re-run logger_create with color argument
    # (will be the same logger since it has the same name)
    bb.msg.logger_create('bitbake-setup', output=sys.stdout,
                         color=args.color,
                         level=logger.getEffectiveLevel())

    if 'func' in args:
        if hasattr(args, 'build_dir'):
            if not os.path.exists(os.path.join(args.build_dir,'build', 'init-build-env')):
                print("Not a valid build directory: build/init-build-env does not exist in {}".format(args.build_dir))
                return

        if not hasattr(args, 'non_interactive'):
            args.non_interactive = True

        builtin_settings = {}
        builtin_settings['default'] = {
                         'top-dir-prefix':os.path.expanduser('~'),
                         'top-dir-name':'bitbake-builds',
                         'registry':default_registry,
                         }

        global_settings = load_settings(global_settings_path(args))
        top_dir = get_top_dir(args, merge_settings(builtin_settings, global_settings, {}, args.cmdline_settings))

        # This cannot be set with the rest of the builtin settings as top_dir needs to be determined first
        builtin_settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads')

        topdir_settings = load_settings(default_settings_path(top_dir))
        all_settings = merge_settings(builtin_settings, global_settings, topdir_settings, args.cmdline_settings)

        if args.func == settings_func:
            settings_func(top_dir, all_settings, args)
            return

        print('Bitbake-setup is using {} as top directory ("bitbake-setup settings --help" shows how to change it).\n'.format(top_dir, global_settings_path(args)))

        d = init_bb_cache(top_dir, all_settings, args)
        args.func(top_dir, all_settings, args, d)
        save_bb_cache()
    else:
        from argparse import Namespace
        parser.print_help()

main()