diff --git a/bitbake/bin/bitbake-setup b/bitbake/bin/bitbake-setup index 7878cd9394..e9e73a9270 100755 --- a/bitbake/bin/bitbake-setup +++ b/bitbake/bin/bitbake-setup @@ -32,9 +32,9 @@ logger = bb.msg.logger_create('bitbake-setup', sys.stdout) def cache_dir(top_dir): return os.path.join(top_dir, '.bitbake-setup-cache') -def init_bb_cache(settings, args): +def init_bb_cache(top_dir, settings, args): dldir = settings["default"]["dl-dir"] - bb_cachedir = os.path.join(cache_dir(args.top_dir), 'bitbake-cache') + bb_cachedir = os.path.join(cache_dir(top_dir), 'bitbake-cache') d = bb.data.init() d.setVar("DL_DIR", dldir) @@ -389,7 +389,7 @@ def choose_fragments(possibilities, parameters, non_interactive, skip_selection) choices[k] = options_enumerated[option_n][1] return choices -def obtain_config(settings, args, source_overrides, d): +def obtain_config(top_dir, settings, args, source_overrides, d): if args.config: config_id = args.config[0] config_parameters = args.config[1:] @@ -407,7 +407,7 @@ def obtain_config(settings, args, source_overrides, d): upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)} else: print("Looking up config {} in configuration registry".format(config_id)) - registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d) + registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d) registry_configs = list_registry(registry_path, with_expired=True) if config_id not in registry_configs: raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id)) @@ -416,7 +416,7 @@ def obtain_config(settings, args, source_overrides, d): if has_expired(expiry_date): print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date)) else: - registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d) + registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d) registry_configs = list_registry(registry_path, with_expired=True) config_id = choose_config(registry_configs, args.non_interactive) config_parameters = [] @@ -429,7 +429,7 @@ def obtain_config(settings, args, source_overrides, d): upstream_config['skip-selection'] = args.skip_selection return upstream_config -def init_config(settings, args, d): +def init_config(top_dir, settings, args, d): stdout = sys.stdout def handle_task_progress(event, d): rate = event.rate if event.rate else '' @@ -437,10 +437,10 @@ def init_config(settings, args, d): print("{}% {} ".format(progress, rate), file=stdout, end='\r') source_overrides = json.load(open(args.source_overrides)) if args.source_overrides else {'sources':{}} - upstream_config = obtain_config(settings, args, source_overrides, d) + upstream_config = obtain_config(top_dir, settings, args, source_overrides, d) print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options']))) - builddir = os.path.join(os.path.abspath(args.top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_"))) + builddir = os.path.join(os.path.abspath(top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_"))) if os.path.exists(os.path.join(builddir, "layers")): print("Build already initialized in {}\nUse 'bitbake-setup status' to check if it needs to be updated or 'bitbake-setup update' to perform the update.".format(builddir)) return @@ -511,7 +511,7 @@ def are_layers_changed(layers, layerdir, d): return changed -def build_status(settings, args, d, update=False): +def build_status(top_dir, settings, args, d, update=False): builddir = args.build_dir confdir = os.path.join(builddir, "config") @@ -523,7 +523,7 @@ def build_status(settings, args, d, update=False): args.non_interactive = True args.skip_selection = current_upstream_config['skip-selection'] source_overrides = current_upstream_config["source-overrides"] - new_upstream_config = obtain_config(settings, args, source_overrides, d) + new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d) write_config(new_upstream_config, confdir) config_diff = bb.process.run('git -C {} diff'.format(confdir))[0] @@ -544,8 +544,8 @@ def build_status(settings, args, d, update=False): print("\nConfiguration in {} has not changed.".format(builddir)) -def build_update(settings, args, d): - build_status(settings, args, d, update=True) +def build_update(top_dir, settings, args, d): + build_status(top_dir, settings, args, d, update=True) def do_fetch(fetcher, dir): # git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch @@ -595,8 +595,8 @@ def list_registry(registry_path, with_expired): json_data[config_name] = {"description": config_desc} return json_data -def list_configs(settings, args, d): - registry_path = update_registry(settings["default"]["registry"], cache_dir(args.top_dir), d) +def list_configs(top_dir, settings, args, d): + registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d) json_data = list_registry(registry_path, args.with_expired) print("\nAvailable configurations:") for config_name, config_data in json_data.items(): @@ -614,7 +614,7 @@ def list_configs(settings, args, d): json.dump(json_data, f, sort_keys=True, indent=4) print("Available configurations written into {}".format(args.write_json)) -def install_buildtools(settings, args, d): +def install_buildtools(top_dir, settings, args, d): buildtools_install_dir = os.path.join(args.build_dir, 'buildtools') if os.path.exists(buildtools_install_dir): if not args.force: @@ -634,117 +634,113 @@ def install_buildtools(settings, args, d): subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True) def default_settings_path(top_dir): - return os.path.join(top_dir, 'bitbake-setup.conf') + return os.path.join(top_dir, 'settings.conf') -def write_settings(top_dir, force_replace, non_interactive=True): +def create_settings(top_dir, non_interactive=True): settings_path = default_settings_path(top_dir) - if not os.path.exists(settings_path) or force_replace: + settings = configparser.ConfigParser() + settings['default'] = { + } + os.makedirs(os.path.dirname(settings_path), exist_ok=True) - settings = configparser.ConfigParser() - settings['default'] = { - 'registry':default_registry, - 'dl-dir':os.path.join(top_dir, '.bitbake-setup-downloads'), - } - os.makedirs(os.path.dirname(settings_path), exist_ok=True) + siteconfpath = os.path.join(top_dir, 'site.conf') + print('A new empty settings file will be created in (you can add settings to it to override defaults from the global settings file)\n {}\n'.format(settings_path)) + print('A common site.conf file will be created, please edit or replace before running builds\n {}\n'.format(siteconfpath)) + if not non_interactive: + y_or_n = input('Bitbake-setup will be configured with the above settings in {}, (y/N): '.format(top_dir)) + if y_or_n != 'y': + print("\nYou can run 'bitbake-setup install-settings' to edit them before setting up builds") + exit() + print() - siteconfpath = os.path.join(top_dir, 'site.conf') - print('Configuration registry set to\n {}\n'.format(settings['default']['registry'])) - print('Bitbake-setup download cache (DL_DIR) set to\n {}\n'.format(settings['default']['dl-dir'])) - print('A new settings file will be created in\n {}\n'.format(settings_path)) - print('A common site.conf file will be created, please edit or replace before running builds\n {}\n'.format(siteconfpath)) - if not non_interactive: - y_or_n = input('Bitbake-setup will be configured with the above settings in {}, (y/N): '.format(top_dir)) - if y_or_n != 'y': - print("\nYou can run 'bitbake-setup install-settings' to edit them before setting up builds") - exit() - print() + if os.path.exists(settings_path): + backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) + os.rename(settings_path, backup_conf) + print("Previous settings are in {}".format(backup_conf)) + with open(settings_path, 'w') as settingsfile: + settings.write(settingsfile) - if os.path.exists(settings_path): - backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) - os.rename(settings_path, backup_conf) - print("Previous settings are in {}".format(backup_conf)) - with open(settings_path, 'w') as settingsfile: - settings.write(settingsfile) - - if os.path.exists(siteconfpath): - backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) - os.rename(siteconfpath, backup_siteconf) - print("Previous settings are in {}".format(backup_siteconf)) - with open(siteconfpath, 'w') as siteconffile: - siteconffile.write('# This file is intended for build host-specific bitbake settings\n') + if os.path.exists(siteconfpath): + backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) + os.rename(siteconfpath, backup_siteconf) + print("Previous settings are in {}".format(backup_siteconf)) + with open(siteconfpath, 'w') as siteconffile: + siteconffile.write('# This file is intended for build host-specific bitbake settings\n') def load_settings(top_dir, non_interactive): - # This creates a new settings file if it does not yet exist - write_settings(top_dir, force_replace=False, non_interactive=non_interactive) - settings_path = default_settings_path(top_dir) + if not os.path.exists(settings_path): + create_settings(top_dir, non_interactive=non_interactive) + settings = configparser.ConfigParser() print('Loading settings from\n {}\n'.format(settings_path)) settings.read_file(open(settings_path)) return settings def global_settings_path(args): - return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'config') + return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf') -def write_global_settings(settings_path, force_replace, non_interactive=True): - if not os.path.exists(settings_path) or force_replace: +def create_global_settings(settings_path, non_interactive=True): + settings = configparser.ConfigParser() + settings['default'] = { + 'top-dir-prefix':os.path.expanduser('~'), + 'top-dir-name':'bitbake-builds', + 'registry':default_registry, + 'dl-dir':os.path.join(os.path.expanduser('~'), '.cache', 'bitbake-setup', 'downloads'), + } + os.makedirs(os.path.dirname(settings_path), exist_ok=True) + print('Configuring global settings in\n {}\n'.format(settings_path)) + print('Top directory prefix (where all top level directories are created) set to\n {}\n'.format(settings['default']['top-dir-prefix'])) + print('Top directory name (this is added to the top directory prefix to form a top directory where builds are set up) set to\n {}\n'.format(settings['default']['top-dir-name'])) + print('Configuration registry set to\n {}\n'.format(settings['default']['registry'])) + print('Bitbake-setup download cache (DL_DIR) set to\n {}\n'.format(settings['default']['dl-dir'])) + if not non_interactive: + y_or_n = input('Write out the global settings as specified above (y/N)? ') + if y_or_n != 'y': + print("\nYou can run 'bitbake-setup install-global-settings' to edit them before setting up builds") + exit() + print() - settings = configparser.ConfigParser() - settings['default'] = { - 'top-dir-prefix':os.path.expanduser('~'), - 'top-dir-name':'bitbake-builds' - } - os.makedirs(os.path.dirname(settings_path), exist_ok=True) - print('Configuring global settings in\n {}\n'.format(settings_path)) - print('Top directory prefix (where all top level directories are created) set to\n {}\n'.format(settings['default']['top-dir-prefix'])) - print('Top directory name (this is added to the top directory prefix to form a top directory where builds are set up) set to\n {}\n'.format(settings['default']['top-dir-name'])) - if not non_interactive: - y_or_n = input('Write out the global settings as specified above (y/N)? ') - if y_or_n != 'y': - print("\nYou can run 'bitbake-setup install-global-settings' to edit them before setting up builds") - exit() - print() - - if os.path.exists(settings_path): - backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) - os.rename(settings_path, backup_conf) - print("Previous global settings are in {}".format(backup_conf)) - with open(settings_path, 'w') as settingsfile: - settings.write(settingsfile) + if os.path.exists(settings_path): + backup_conf = settings_path + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S")) + os.rename(settings_path, backup_conf) + print("Previous global settings are in {}".format(backup_conf)) + with open(settings_path, 'w') as settingsfile: + settings.write(settingsfile) def load_global_settings(settings_path, non_interactive): - # This creates a new settings file if it does not yet exist - write_global_settings(settings_path, force_replace=False, non_interactive=non_interactive) + if not os.path.exists(settings_path): + create_global_settings(settings_path, non_interactive=non_interactive) settings = configparser.ConfigParser() print('Loading global settings from\n {}\n'.format(settings_path)) settings.read_file(open(settings_path)) return settings -def change_settings(top_dir, new_settings): - settings = load_settings(top_dir, non_interactive=True) - for section, section_settings in new_settings.items(): - for setting, value in section_settings.items(): - settings[section][setting] = value - print("Setting '{}' in section '{}' is changed to '{}'".format(setting, section, value)) +def change_setting(settings_path, settings, args): + if args.section and args.key and args.value: + settings[args.section][args.key] = args.value + print("Setting '{}' in section '{}' is changed to '{}'".format(args.key, args.section, args.value)) + if args.unset: + section = args.unset[0] + setting = args.unset[1] + if section in settings.keys() and setting in settings[section].keys(): + del settings[section][setting] + print("Setting '{} in section '{}' is removed".format(setting, section)) - settings_path = default_settings_path(top_dir) with open(settings_path, 'w') as settingsfile: settings.write(settingsfile) print("New settings written to {}".format(settings_path)) - return settings -def change_global_settings(settings_path, new_settings): - settings = load_global_settings(settings_path, non_interactive=True) - for section, section_settings in new_settings.items(): - for setting, value in section_settings.items(): - settings[section][setting] = value - print("Setting '{}' in section '{}' is changed to '{}'".format(setting, section, value)) +def setting_global(args): + settings = load_global_settings(global_settings_path(args), args.non_interactive) + settings_path = global_settings_path(args) + change_setting(settings_path, settings, args) - with open(settings_path, 'w') as settingsfile: - settings.write(settingsfile) - print("New global settings written to {}".format(settings_path)) - return settings +def setting(top_dir, args): + settings = load_settings(top_dir, args.non_interactive) + settings_path = default_settings_path(top_dir) + change_setting(settings_path, settings, args) def get_build_dir_via_bbpath(): bbpath = os.environ.get('BBPATH') @@ -755,7 +751,7 @@ def get_build_dir_via_bbpath(): return build_dir return None -def get_top_dir(args, global_settings): +def get_top_dir(args, settings): build_dir_via_bbpath = get_build_dir_via_bbpath() if build_dir_via_bbpath: top_dir = os.path.dirname(build_dir_via_bbpath) @@ -763,20 +759,25 @@ def get_top_dir(args, global_settings): return top_dir if hasattr(args, 'build_dir'): - # commands without --top-dir-prefix/name arguments (status, update) still need to know where - # the top dir is, but it should be auto-deduced as parent of args.build_dir top_dir = os.path.dirname(os.path.normpath(args.build_dir)) return top_dir - top_dir_prefix = args.top_dir_prefix if args.top_dir_prefix else global_settings['default']['top-dir-prefix'] - top_dir_name = args.top_dir_name if args.top_dir_name else global_settings['default']['top-dir-name'] + top_dir_prefix = settings['default']['top-dir-prefix'] + top_dir_name = settings['default']['top-dir-name'] return os.path.join(top_dir_prefix, top_dir_name) -def main(): - def add_top_dir_arg(parser): - parser.add_argument('--top-dir-prefix', help='Top level directory prefix. This is where all top level directories are created.') - parser.add_argument('--top-dir-name', help='Top level directory name. Together with the top directory prefix this forms a top directory where builds are set up and downloaded configurations and layers are cached for reproducibility and offline builds.') +def merge_settings(global_settings, local_settings, cmdline_settings): + all_settings = global_settings + for section, section_settings in local_settings.items(): + for setting, value in section_settings.items(): + all_settings[section][setting] = value + for (section, setting, value) in cmdline_settings: + all_settings[section][setting] = value + + return all_settings + +def main(): def add_build_dir_arg(parser): build_dir = get_build_dir_via_bbpath() if build_dir: @@ -792,18 +793,17 @@ def main(): parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.') - parser.add_argument('--global-settings', action='store', help='Path to the global settings file where defaults for top directory prefix and name can be specified') + parser.add_argument('--global-settings', action='store', help='Path to the global settings file.') + parser.add_argument('--setting', default=[], action='append', nargs=3, help='Modify a setting (for this bitbake-setup invocation only), for example "--setting default top-dir-prefix /path/to/top/dir".') subparsers = parser.add_subparsers() parser_list = subparsers.add_parser('list', help='List available configurations') - add_top_dir_arg(parser_list) parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.') parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.') parser_list.set_defaults(func=list_configs) parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it') - add_top_dir_arg(parser_init) parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.") parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.') parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.') @@ -824,25 +824,16 @@ def main(): parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.') parser_install_buildtools.set_defaults(func=install_buildtools) - parser_install_settings = subparsers.add_parser('install-settings', help='Write a settings file with default values into the top level directory (contains the location of build configuration registry, downloads directory and other settings specific to a top directory)') - add_top_dir_arg(parser_install_settings) - parser_install_settings.set_defaults(func=write_settings) + parser_install_global_settings = subparsers.add_parser('install-global-settings', help='Write a global settings file with default values') + parser_install_global_settings.set_defaults(func=create_global_settings) - parser_install_global_settings = subparsers.add_parser('install-global-settings', help='Write a global settings file with default values (contains the default prefix and name of the top directory)') - parser_install_global_settings.set_defaults(func=write_global_settings) - - parser_change_setting = subparsers.add_parser('change-setting', help='Change a setting in the settings file') - add_top_dir_arg(parser_change_setting) - parser_change_setting.add_argument('section', help="Section in a settings file, typically 'default'") - parser_change_setting.add_argument('key', help="Name of the setting") - parser_change_setting.add_argument('value', help="Value of the setting") - parser_change_setting.set_defaults(func=change_settings) - - parser_change_global_setting = subparsers.add_parser('change-global-setting', help='Change a setting in the global settings file') - parser_change_global_setting.add_argument('section', help="Section in a global settings file, typically 'default'") - parser_change_global_setting.add_argument('key', help="Name of the setting") - parser_change_global_setting.add_argument('value', help="Value of the setting") - parser_change_global_setting.set_defaults(func=change_global_settings) + parser_setting = subparsers.add_parser('setting', help='Set or unset a setting in a setting file (e.g. the default prefix and name of the top directory, the location of build configuration registry, downloads directory and other settings specific to a top directory)') + parser_setting.add_argument('section', nargs='?', help="Section in a settings file, typically 'default'") + parser_setting.add_argument('key', nargs='?', help="Name of the setting") + parser_setting.add_argument('value', nargs='?', help="Value of the setting") + parser_setting.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory") + parser_setting.add_argument('--unset', nargs=2, help="Unset a setting, e.g. 'bitbake-setup setting --unset default registry' would revert to the registry setting in a global settings file") + parser_setting.set_defaults(func=setting) args = parser.parse_args() @@ -859,11 +850,8 @@ def main(): level=logger.getEffectiveLevel()) if 'func' in args: - if args.func == write_global_settings: - write_global_settings(global_settings_path(args), force_replace=True) - return - elif args.func == change_global_settings: - change_global_settings(global_settings_path(args), {args.section:{args.key:args.value}}) + if args.func == create_global_settings: + create_global_settings(global_settings_path(args)) return if hasattr(args, 'build_dir'): @@ -874,19 +862,24 @@ def main(): if not hasattr(args, 'non_interactive'): args.non_interactive = True - global_settings = load_global_settings(global_settings_path(args), args.non_interactive) - args.top_dir = get_top_dir(args, global_settings) + if args.func == setting and vars(args)['global']: + setting_global(args) + return - print('Bitbake-setup is using {} as top directory (can be changed with --top-dir-prefix/name arguments or by setting them in {}).\n'.format(args.top_dir, global_settings_path(args))) - if args.func == write_settings: - write_settings(args.top_dir, force_replace=True) - elif args.func == change_settings: - change_settings(args.top_dir, {args.section:{args.key:args.value}}) - else: - settings = load_settings(args.top_dir, args.non_interactive) - d = init_bb_cache(settings, args) - args.func(settings, args, d) - save_bb_cache() + global_settings = load_global_settings(global_settings_path(args), args.non_interactive) + top_dir = get_top_dir(args, merge_settings(global_settings, {}, args.setting)) + + if args.func == setting: + setting(top_dir, args) + return + + print('Bitbake-setup is using {} as top directory (can be changed by setting top dir prefix and name in {}).\n'.format(top_dir, global_settings_path(args))) + + settings = load_settings(top_dir, args.non_interactive) + settings = merge_settings(global_settings, settings, args.setting) + d = init_bb_cache(top_dir, settings, args) + args.func(top_dir, settings, args, d) + save_bb_cache() else: from argparse import Namespace parser.print_help() diff --git a/bitbake/lib/bb/tests/setup.py b/bitbake/lib/bb/tests/setup.py index 495d1da203..22edda40e4 100644 --- a/bitbake/lib/bb/tests/setup.py +++ b/bitbake/lib/bb/tests/setup.py @@ -235,16 +235,18 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) out = self.runbbsetup("install-global-settings") settings_path = "{}/global-config".format(self.tempdir) self.assertIn(settings_path, out[0]) - out = self.runbbsetup("change-global-setting default top-dir-prefix {}".format(self.tempdir)) + out = self.runbbsetup("setting --global default top-dir-prefix {}".format(self.tempdir)) self.assertIn("Setting 'top-dir-prefix' in section 'default' is changed to", out[0]) - self.assertIn("New global settings written to".format(settings_path), out[0]) + self.assertIn("New settings written to".format(settings_path), out[0]) + out = self.runbbsetup("setting --global default dl-dir {}".format(os.path.join(self.tempdir, 'downloads'))) + self.assertIn("Setting 'dl-dir' in section 'default' is changed to", out[0]) + self.assertIn("New settings written to".format(settings_path), out[0]) # check that writing settings works and then adjust them to point to # test registry repo - out = self.runbbsetup("install-settings") - settings_path = "{}/bitbake-builds/bitbake-setup.conf".format(self.tempdir) + out = self.runbbsetup("setting default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)) + settings_path = "{}/bitbake-builds/settings.conf".format(self.tempdir) self.assertIn(settings_path, out[0]) - out = self.runbbsetup("change-setting default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)) self.assertIn("Setting 'registry' in section 'default' is changed to", out[0]) self.assertIn("New settings written to".format(settings_path), out[0])