diff options
| -rw-r--r-- | lib/bb/command.py | 195 | ||||
| -rw-r--r-- | lib/bb/cooker.py | 19 | ||||
| -rwxr-xr-x | lib/bb/main.py | 83 | ||||
| -rw-r--r-- | lib/bb/remotedata.py | 74 | ||||
| -rw-r--r-- | lib/bb/tinfoil.py | 385 | ||||
| -rw-r--r-- | lib/bblayers/query.py | 28 |
6 files changed, 665 insertions, 119 deletions
diff --git a/lib/bb/command.py b/lib/bb/command.py index 012b35faf..d5be86dab 100644 --- a/lib/bb/command.py +++ b/lib/bb/command.py @@ -28,8 +28,15 @@ and must not trigger events, directly or indirectly. Commands are queued in a CommandQueue """ +from collections import OrderedDict, defaultdict + import bb.event import bb.cooker +import bb.remotedata + +class DataStoreConnectionHandle(object): + def __init__(self, dsindex=0): + self.dsindex = dsindex class CommandCompleted(bb.event.Event): pass @@ -55,6 +62,7 @@ class Command: self.cooker = cooker self.cmds_sync = CommandsSync() self.cmds_async = CommandsAsync() + self.remotedatastores = bb.remotedata.RemoteDatastores(cooker) # FIXME Add lock for this self.currentAsyncCommand = None @@ -298,6 +306,193 @@ class CommandsSync: command.cooker.updateConfigOpts(options, environment) updateConfig.needconfig = False + def parseConfiguration(self, command, params): + """Instruct bitbake to parse its configuration + NOTE: it is only necessary to call this if you aren't calling any normal action + (otherwise parsing is taken care of automatically) + """ + command.cooker.parseConfiguration() + parseConfiguration.needconfig = False + + def getLayerPriorities(self, command, params): + ret = [] + # regex objects cannot be marshalled by xmlrpc + for collection, pattern, regex, pri in command.cooker.bbfile_config_priorities: + ret.append((collection, pattern, regex.pattern, pri)) + return ret + getLayerPriorities.readonly = True + + def getRecipes(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.recipecaches[mc].pkg_pn.items()) + getRecipes.readonly = True + + def getRecipeDepends(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.recipecaches[mc].deps.items()) + getRecipeDepends.readonly = True + + def getRecipeVersions(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.recipecaches[mc].pkg_pepvpr + getRecipeVersions.readonly = True + + def getRuntimeDepends(self, command, params): + ret = [] + try: + mc = params[0] + except IndexError: + mc = '' + rundeps = command.cooker.recipecaches[mc].rundeps + for key, value in rundeps.items(): + if isinstance(value, defaultdict): + value = dict(value) + ret.append((key, value)) + return ret + getRuntimeDepends.readonly = True + + def getRuntimeRecommends(self, command, params): + ret = [] + try: + mc = params[0] + except IndexError: + mc = '' + runrecs = command.cooker.recipecaches[mc].runrecs + for key, value in runrecs.items(): + if isinstance(value, defaultdict): + value = dict(value) + ret.append((key, value)) + return ret + getRuntimeRecommends.readonly = True + + def getRecipeInherits(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.recipecaches[mc].inherits + getRecipeInherits.readonly = True + + def getBbFilePriority(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.recipecaches[mc].bbfile_priority + getBbFilePriority.readonly = True + + def getDefaultPreference(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.recipecaches[mc].pkg_dp + getDefaultPreference.readonly = True + + def getSkippedRecipes(self, command, params): + # Return list sorted by reverse priority order + import bb.cache + skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), + key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + return list(skipdict.items()) + getSkippedRecipes.readonly = True + + def getOverlayedRecipes(self, command, params): + return list(command.cooker.collection.overlayed.items()) + getOverlayedRecipes.readonly = True + + def getFileAppends(self, command, params): + fn = params[0] + return command.cooker.collection.get_file_appends(fn) + getFileAppends.readonly = True + + def getAllAppends(self, command, params): + return command.cooker.collection.bbappends + getAllAppends.readonly = True + + def findProviders(self, command, params): + return command.cooker.findProviders() + findProviders.readonly = True + + def findBestProvider(self, command, params): + pn = params[0] + return command.cooker.findBestProvider(pn) + findBestProvider.readonly = True + + def allProviders(self, command, params): + try: + mc = params[0] + except IndexError: + mc = '' + return list(bb.providers.allProviders(command.cooker.recipecaches[mc]).items()) + allProviders.readonly = True + + def getRuntimeProviders(self, command, params): + rprovide = params[0] + try: + mc = params[1] + except IndexError: + mc = '' + all_p = bb.providers.getRuntimeProviders(command.cooker.recipecaches[mc], rprovide) + if all_p: + best = bb.providers.filterProvidersRunTime(all_p, rprovide, + command.cooker.data, + command.cooker.recipecaches[mc])[0][0] + else: + best = None + return all_p, best + getRuntimeProviders.readonly = True + + def dataStoreConnectorFindVar(self, command, params): + dsindex = params[0] + name = params[1] + datastore = command.remotedatastores[dsindex] + value = datastore._findVar(name) + + if value: + content = value.get('_content', None) + if isinstance(content, bb.data_smart.DataSmart): + # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully + idx = command.remotedatastores.check_store(content, True) + return {'_content': DataStoreConnectionHandle(idx), '_connector_origtype': 'DataStoreConnectionHandle'} + elif isinstance(content, set): + return {'_content': list(content), '_connector_origtype': 'set'} + return value + dataStoreConnectorFindVar.readonly = True + + def dataStoreConnectorGetKeys(self, command, params): + dsindex = params[0] + datastore = command.remotedatastores[dsindex] + return list(datastore.keys()) + dataStoreConnectorGetKeys.readonly = True + + def dataStoreConnectorGetVarHistory(self, command, params): + dsindex = params[0] + name = params[1] + datastore = command.remotedatastores[dsindex] + return datastore.varhistory.variable(name) + dataStoreConnectorGetVarHistory.readonly = True + + def dataStoreConnectorExpandPythonRef(self, command, params): + dsindex = params[0] + varname = params[1] + expr = params[2] + if dsindex: + datastore = self.dataStores[dsindex] + else: + datastore = command.cooker.data + varparse = bb.data_smart.VariableParse(varname, datastore) + return varparse.python_sub(expr) + class CommandsAsync: """ A class of asynchronous commands diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 2614c4485..48904a52d 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -583,13 +583,12 @@ class BBCooker: def showVersions(self): - pkg_pn = self.recipecaches[''].pkg_pn - (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn) + (latest_versions, preferred_versions) = self.findProviders() logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") - for p in sorted(pkg_pn): + for p in sorted(self.recipecaches[''].pkg_pn): pref = preferred_versions[p] latest = latest_versions[p] @@ -1084,6 +1083,20 @@ class BBCooker: if matches: bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) + def findProviders(self, mc=''): + return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn) + + def findBestProvider(self, pn, mc=''): + if pn in self.recipecaches[mc].providers: + filenames = self.recipecaches[mc].providers[pn] + eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.expanded_data, self.recipecaches[mc]) + filename = eligible[0] + return None, None, None, filename + elif pn in self.recipecaches[mc].pkg_pn: + return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn) + else: + return None, None, None, None + def findConfigFiles(self, varname): """ Find config files which are appropriate values for varname. diff --git a/lib/bb/main.py b/lib/bb/main.py index a544c0aec..443f5ec2f 100755 --- a/lib/bb/main.py +++ b/lib/bb/main.py @@ -389,12 +389,8 @@ def bitbake_main(configParams, configuration): except: pass - configuration.setConfigParameters(configParams) - ui_module = import_extension_module(bb.ui, configParams.ui, 'main') - servermodule = import_extension_module(bb.server, configParams.servertype, 'BitBakeServer') - if configParams.server_only: if configParams.servertype != "xmlrpc": raise BBMainException("FATAL: If '--server-only' is defined, we must set the " @@ -442,6 +438,31 @@ def bitbake_main(configParams, configuration): bb.msg.init_msgconfig(configParams.verbose, configuration.debug, configuration.debug_domains) + server, server_connection, ui_module = setup_bitbake(configParams, configuration) + if server_connection is None and configParams.kill_server: + return 0 + + if not configParams.server_only: + if configParams.status_only: + server_connection.terminate() + return 0 + + try: + return ui_module.main(server_connection.connection, server_connection.events, + configParams) + finally: + bb.event.ui_queue = [] + server_connection.terminate() + else: + print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host, + server.serverImpl.port)) + if configParams.foreground: + server.serverImpl.serve_forever() + return 0 + + return 1 + +def setup_bitbake(configParams, configuration, extrafeatures=None): # Ensure logging messages get sent to the UI as events handler = bb.event.LogHandler() if not configParams.status_only: @@ -451,8 +472,11 @@ def bitbake_main(configParams, configuration): # Clear away any spurious environment variables while we stoke up the cooker cleanedvars = bb.utils.clean_environment() - featureset = [] - if not configParams.server_only: + if configParams.server_only: + featureset = [] + ui_module = None + else: + ui_module = import_extension_module(bb.ui, configParams.ui, 'main') # Collect the feature set for the UI featureset = getattr(ui_module, "featureSet", []) @@ -463,11 +487,15 @@ def bitbake_main(configParams, configuration): setattr(configuration, "%s_server" % param, value) param = "%s_server" % param - if not configParams.remote_server: - # we start a server with a given configuration - server = start_server(servermodule, configParams, configuration, featureset) - bb.event.ui_queue = [] - else: + if extrafeatures: + for feature in extrafeatures: + if not feature in featureset: + featureset.append(feature) + + servermodule = import_extension_module(bb.server, + configParams.servertype, + 'BitBakeServer') + if configParams.remote_server: if os.getenv('BBSERVER') == 'autostart': if configParams.remote_server == 'autostart' or \ not servermodule.check_connection(configParams.remote_server, timeout=2): @@ -475,14 +503,19 @@ def bitbake_main(configParams, configuration): srv = start_server(servermodule, configParams, configuration, featureset) configParams.remote_server = '%s:%d' % tuple(configuration.interface) bb.event.ui_queue = [] - # we start a stub server that is actually a XMLRPClient that connects to a real server + from bb.server.xmlrpc import BitBakeXMLRPCClient server = servermodule.BitBakeXMLRPCClient(configParams.observe_only, configParams.xmlrpctoken) server.saveConnectionDetails(configParams.remote_server) + else: + # we start a server with a given configuration + server = start_server(servermodule, configParams, configuration, featureset) + bb.event.ui_queue = [] - - if not configParams.server_only: + if configParams.server_only: + server_connection = None + else: try: server_connection = server.establishConnection(featureset) except Exception as e: @@ -491,7 +524,7 @@ def bitbake_main(configParams, configuration): if configParams.kill_server: server_connection.connection.terminateServer() bb.event.ui_queue = [] - return 0 + return None, None, None server_connection.setupEventQueue() @@ -501,22 +534,4 @@ def bitbake_main(configParams, configuration): logger.removeHandler(handler) - - if configParams.status_only: - server_connection.terminate() - return 0 - - try: - return ui_module.main(server_connection.connection, server_connection.events, - configParams) - finally: - bb.event.ui_queue = [] - server_connection.terminate() - else: - print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host, - server.serverImpl.port)) - if configParams.foreground: - server.serverImpl.serve_forever() - return 0 - - return 1 + return server, server_connection, ui_module diff --git a/lib/bb/remotedata.py b/lib/bb/remotedata.py new file mode 100644 index 000000000..932ee430e --- /dev/null +++ b/lib/bb/remotedata.py @@ -0,0 +1,74 @@ +""" +BitBake 'remotedata' module + +Provides support for using a datastore from the bitbake client +""" + +# Copyright (C) 2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import bb.data + +class RemoteDatastores: + """Used on the server side to manage references to server-side datastores""" + def __init__(self, cooker): + self.cooker = cooker + self.datastores = {} + self.locked = [] + self.nextindex = 1 + + def __len__(self): + return len(self.datastores) + + def __getitem__(self, key): + if key is None: + return self.cooker.data + else: + return self.datastores[key] + + def items(self): + return self.datastores.items() + + def store(self, d, locked=False): + """ + Put a datastore into the collection. If locked=True then the datastore + is understood to be managed externally and cannot be released by calling + release(). + """ + idx = self.nextindex + self.datastores[idx] = d + if locked: + self.locked.append(idx) + self.nextindex += 1 + return idx + + def check_store(self, d, locked=False): + """ + Put a datastore into the collection if it's not already in there; + in either case return the index + """ + for key, val in self.datastores.items(): + if val is d: + idx = key + break + else: + idx = self.store(d, locked) + return idx + + def release(self, idx): + """Discard a datastore in the collection""" + if idx in self.locked: + raise Exception('Tried to release locked datastore %d' % idx) + del self.datastores[idx] diff --git a/lib/bb/tinfoil.py b/lib/bb/tinfoil.py index 8899e861c..459f6c128 100644 --- a/lib/bb/tinfoil.py +++ b/lib/bb/tinfoil.py @@ -1,6 +1,6 @@ # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities # -# Copyright (C) 2012 Intel Corporation +# Copyright (C) 2012-2016 Intel Corporation # Copyright (C) 2011 Mentor Graphics Corporation # # This program is free software; you can redistribute it and/or modify @@ -17,47 +17,172 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import logging -import warnings import os import sys +import atexit +import re +from collections import OrderedDict, defaultdict import bb.cache import bb.cooker import bb.providers import bb.utils -from bb.cooker import state, BBCooker, CookerFeatures +import bb.command from bb.cookerdata import CookerConfiguration, ConfigParameters +from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException import bb.fetch2 + +# We need this in order to shut down the connection to the bitbake server, +# otherwise the process will never properly exit +_server_connections = [] +def _terminate_connections(): + for connection in _server_connections: + connection.terminate() +atexit.register(_terminate_connections) + +class TinfoilUIException(Exception): + """Exception raised when the UI returns non-zero from its main function""" + def __init__(self, returncode): + self.returncode = returncode + def __repr__(self): + return 'UI module main returned %d' % self.returncode + +class TinfoilCommandFailed(Exception): + """Exception raised when run_command fails""" + +class TinfoilDataStoreConnector: + + def __init__(self, tinfoil, dsindex): + self.tinfoil = tinfoil + self.dsindex = dsindex + def getVar(self, name): + value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name) + if isinstance(value, dict): + if '_connector_origtype' in value: + value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype']) + del value['_connector_origtype'] + + return value + def getKeys(self): + return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex)) + def getVarHistory(self, name): + return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name) + def expandPythonRef(self, varname, expr): + ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', self.dsindex, varname, expr) + return ret + def setVar(self, varname, value): + if self.dsindex is None: + self.tinfoil.run_command('setVariable', varname, value) + else: + # Not currently implemented - indicate that setting should + # be redirected to local side + return True + +class TinfoilCookerAdapter: + """ + Provide an adapter for existing code that expects to access a cooker object via Tinfoil, + since now Tinfoil is on the client side it no longer has direct access. + """ + + class TinfoilCookerCollectionAdapter: + """ cooker.collection adapter """ + def __init__(self, tinfoil): + self.tinfoil = tinfoil + def get_file_appends(self, fn): + return self.tinfoil.run_command('getFileAppends', fn) + def __getattr__(self, name): + if name == 'overlayed': + return self.tinfoil.get_overlayed_recipes() + elif name == 'bbappends': + return self.tinfoil.run_command('getAllAppends') + else: + raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) + + class TinfoilRecipeCacheAdapter: + """ cooker.recipecache adapter """ + def __init__(self, tinfoil): + self.tinfoil = tinfoil + self._cache = {} + + def get_pkg_pn_fn(self): + pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or []) + pkg_fn = {} + for pn, fnlist in pkg_pn.items(): + for fn in fnlist: + pkg_fn[fn] = pn + self._cache['pkg_pn'] = pkg_pn + self._cache['pkg_fn'] = pkg_fn + + def __getattr__(self, name): + # Grab these only when they are requested since they aren't always used + if name in self._cache: + return self._cache[name] + elif name == 'pkg_pn': + self.get_pkg_pn_fn() + return self._cache[name] + elif name == 'pkg_fn': + self.get_pkg_pn_fn() + return self._cache[name] + elif name == 'deps': + attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or []) + elif name == 'rundeps': + attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or []) + elif name == 'runrecs': + attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or []) + elif name == 'pkg_pepvpr': + attrvalue = self.tinfoil.run_command('getRecipeVersions') or {} + elif name == 'inherits': + attrvalue = self.tinfoil.run_command('getRecipeInherits') or {} + elif name == 'bbfile_priority': + attrvalue = self.tinfoil.run_command('getBbFilePriority') or {} + elif name == 'pkg_dp': + attrvalue = self.tinfoil.run_command('getDefaultPreference') or {} + else: + raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) + + self._cache[name] = attrvalue + return attrvalue + + def __init__(self, tinfoil): + self.tinfoil = tinfoil + self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.recipecaches = {} + # FIXME all machines + self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) + self._cache = {} + def __getattr__(self, name): + # Grab these only when they are requested since they aren't always used + if name in self._cache: + return self._cache[name] + elif name == 'skiplist': + attrvalue = self.tinfoil.get_skipped_recipes() + elif name == 'bbfile_config_priorities': + ret = self.tinfoil.run_command('getLayerPriorities') + bbfile_config_priorities = [] + for collection, pattern, regex, pri in ret: + bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri)) + + attrvalue = bbfile_config_priorities + else: + raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) + + self._cache[name] = attrvalue + return attrvalue + + def findBestProvider(self, pn): + return self.tinfoil.find_best_provider(pn) + + class Tinfoil: - def __init__(self, output=sys.stdout, tracking=False): - # Needed to avoid deprecation warnings with python 2.6 - warnings.filterwarnings("ignore", category=DeprecationWarning) - # Set up logging + def __init__(self, output=sys.stdout, tracking=False): self.logger = logging.getLogger('BitBake') - self._log_hdlr = logging.StreamHandler(output) - bb.msg.addDefaultlogFilter(self._log_hdlr) - format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") - if output.isatty(): - format.enable_color() - self._log_hdlr.setFormatter(format) - self.logger.addHandler(self._log_hdlr) - - self.config = CookerConfiguration() - configparams = TinfoilConfigParameters(parse_only=True) - self.config.setConfigParameters(configparams) - self.config.setServerRegIdleCallback(self.register_idle_function) - features = [] - if tracking: - features.append(CookerFeatures.BASEDATASTORE_TRACKING) - self.cooker = BBCooker(self.config, features) - self.config_data = self.cooker.data - bb.providers.logger.setLevel(logging.ERROR) - self.cooker_data = None - - def register_idle_function(self, function, data): - pass + self.config_data = None + self.cooker = None + self.tracking = tracking + self.ui_module = None + self.server_connection = None def __enter__(self): return self @@ -65,30 +190,120 @@ class Tinfoil: def __exit__(self, type, value, traceback): self.shutdown() - def parseRecipes(self): - sys.stderr.write("Parsing recipes..") - self.logger.setLevel(logging.WARNING) + def prepare(self, config_only=False, config_params=None, quiet=0): + if self.tracking: + extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING] + else: + extrafeatures = [] - try: - while self.cooker.state in (state.initial, state.parsing): - self.cooker.updateCache() - except KeyboardInterrupt: - self.cooker.shutdown() - self.cooker.updateCache() - sys.exit(2) + if not config_params: + config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet) - self.logger.setLevel(logging.INFO) - sys.stderr.write("done.\n") + cookerconfig = CookerConfiguration() + cookerconfig.setConfigParameters(config_params) - self.cooker_data = self.cooker.recipecaches[''] + server, self.server_connection, ui_module = setup_bitbake(config_params, + cookerconfig, + extrafeatures) - def prepare(self, config_only = False): - if not self.cooker_data: + self.ui_module = ui_module + + if self.server_connection: + _server_connections.append(self.server_connection) if config_only: - self.cooker.parseConfiguration() - self.cooker_data = self.cooker.recipecaches[''] + config_params.updateToServer(self.server_connection.connection, os.environ.copy()) + self.run_command('parseConfiguration') else: - self.parseRecipes() + self.run_actions(config_params) + + self.config_data = bb.data.init() + connector = TinfoilDataStoreConnector(self, None) + self.config_data.setVar('_remote_data', connector) + self.cooker = TinfoilCookerAdapter(self) + self.cooker_data = self.cooker.recipecaches[''] + else: + raise Exception('Failed to start bitbake server') + + def run_actions(self, config_params): + """ + Run the actions specified in config_params through the UI. + """ + ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params) + if ret: + raise TinfoilUIException(ret) + + def parseRecipes(self): + """ + Force a parse of all recipes. Normally you should specify + config_only=False when calling prepare() instead of using this + function; this function is designed for situations where you need + to initialise Tinfoil and use it with config_only=True first and + then conditionally call this function to parse recipes later. + """ + config_params = TinfoilConfigParameters(config_only=False) + self.run_actions(config_params) + + def run_command(self, command, *params): + """ + Run a command on the server (as implemented in bb.command). + Note that there are two types of command - synchronous and + asynchronous; in order to receive the results of asynchronous + commands you will need to set an appropriate event mask + using set_event_mask() and listen for the result using + wait_event() - with the correct event mask you'll at least get + bb.command.CommandCompleted and possibly other events before + that depending on the command. + """ + if not self.server_connection: + raise Exception('Not connected to server (did you call .prepare()?)') + + commandline = [command] + if params: + commandline.extend(params) + result = self.server_connection.connection.runCommand(commandline) + if result[1]: + raise TinfoilCommandFailed(result[1]) + return result[0] + + def set_event_mask(self, eventlist): + """Set the event mask which will be applied within wait_event()""" + if not self.server_connection: + raise Exception('Not connected to server (did you call .prepare()?)') + llevel, debug_domains = bb.msg.constructLogOptions() + ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist) + if not ret: + raise Exception('setEventMask failed') + + def wait_event(self, timeout=0): + """ + Wait for an event from the server for the specified time. + A timeout of 0 means don't wait if there are no events in the queue. + Returns the next event in the queue or None if the timeout was + reached. Note that in order to recieve any events you will + first need to set the internal event mask using set_event_mask() + (otherwise whatever event mask the UI set up will be in effect). + """ + if not self.server_connection: + raise Exception('Not connected to server (did you call .prepare()?)') + return self.server_connection.events.waitEvent(timeout) + + def get_overlayed_recipes(self): + return defaultdict(list, self.run_command('getOverlayedRecipes')) + + def get_skipped_recipes(self): + return OrderedDict(self.run_command('getSkippedRecipes')) + + def get_all_providers(self): + return defaultdict(list, self.run_command('allProviders')) + + def find_providers(self): + return self.run_command('findProviders') + + def find_best_provider(self, pn): + return self.run_command('findBestProvider', pn) + + def get_runtime_providers(self, rdep): + return self.run_command('getRuntimeProviders', rdep) def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): """ @@ -126,22 +341,72 @@ class Tinfoil: envdata = parser.loadDataFull(fn, appendfiles) return envdata + def build_file(self, buildfile, task): + """ + Runs the specified task for just a single recipe (i.e. no dependencies). + This is equivalent to bitbake -b. + """ + return self.run_command('buildFile', buildfile, task) + def shutdown(self): - self.cooker.shutdown(force=True) - self.cooker.post_serve() - self.cooker.unlockBitbake() - self.logger.removeHandler(self._log_hdlr) + if self.server_connection: + self.run_command('clientComplete') + _server_connections.remove(self.server_connection) + bb.event.ui_queue = [] + self.server_connection.terminate() + self.server_connection = None -class TinfoilConfigParameters(ConfigParameters): + def _reconvert_type(self, obj, origtypename): + """ + Convert an object back to the right type, in the case + that marshalling has changed it (especially with xmlrpc) + """ + supported_types = { + 'set': set, + 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle, + } - def __init__(self, **options): + origtype = supported_types.get(origtypename, None) + if origtype is None: + raise Exception('Unsupported type "%s"' % origtypename) + if type(obj) == origtype: + newobj = obj + elif isinstance(obj, dict): + # New style class + newobj = origtype() + for k,v in obj.items(): + setattr(newobj, k, v) + else: + # Assume we can coerce the type + newobj = origtype(obj) + + if isinstance(newobj, bb.command.DataStoreConnectionHandle): + connector = TinfoilDataStoreConnector(self, newobj.dsindex) + newobj = bb.data.init() + newobj.setVar('_remote_data', connector) + + return newobj + + +class TinfoilConfigParameters(BitBakeConfigParameters): + + def __init__(self, config_only, **options): self.initial_options = options - super(TinfoilConfigParameters, self).__init__() + # Apply some sane defaults + if not 'parse_only' in options: + self.initial_options['parse_only'] = not config_only + #if not 'status_only' in options: + # self.initial_options['status_only'] = config_only + if not 'ui' in options: + self.initial_options['ui'] = 'knotty' + if not 'argv' in options: + self.initial_options['argv'] = [] - def parseCommandLine(self, argv=sys.argv): - class DummyOptions: - def __init__(self, initial_options): - for key, val in initial_options.items(): - setattr(self, key, val) + super(TinfoilConfigParameters, self).__init__() - return DummyOptions(self.initial_options), None + def parseCommandLine(self, argv=None): + # We don't want any parameters parsed from the command line + opts = super(TinfoilConfigParameters, self).parseCommandLine([]) + for key, val in self.initial_options.items(): + setattr(opts[0], key, val) + return opts diff --git a/lib/bblayers/query.py b/lib/bblayers/query.py index 29491163c..5def7179c 100644 --- a/lib/bblayers/query.py +++ b/lib/bblayers/query.py @@ -5,8 +5,6 @@ import sys import os import re -import bb.cache -import bb.providers import bb.utils from bblayers.common import LayerPlugin @@ -122,15 +120,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix. sys.exit(1) pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn - (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecaches[''], pkg_pn) - allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecaches['']) + (latest_versions, preferred_versions) = self.tinfoil.find_providers() + allproviders = self.tinfoil.get_all_providers() # Ensure we list skipped recipes # We are largely guessing about PN, PV and the preferred version here, # but we have no choice since skipped recipes are not fully parsed skiplist = list(self.tinfoil.cooker.skiplist.keys()) - skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) ) - skiplist.reverse() for fn in skiplist: recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') p = recipe_parts[0] @@ -265,10 +261,7 @@ Lists recipes with the bbappends that apply to them as subitems. def show_appends_for_pn(self, pn): filenames = self.tinfoil.cooker_data.pkg_pn[pn] - best = bb.providers.findBestProvider(pn, - self.tinfoil.config_data, - self.tinfoil.cooker_data, - self.tinfoil.cooker_data.pkg_pn) + best = self.tinfoil.find_best_provider(pn) best_filename = os.path.basename(best[3]) return self.show_appends_output(filenames, best_filename) @@ -336,10 +329,7 @@ NOTE: .bbappend files can impact the dependencies. deps = self.tinfoil.cooker_data.deps[f] for pn in deps: if pn in self.tinfoil.cooker_data.pkg_pn: - best = bb.providers.findBestProvider(pn, - self.tinfoil.config_data, - self.tinfoil.cooker_data, - self.tinfoil.cooker_data.pkg_pn) + best = self.tinfoil.find_best_provider(pn) self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) # The RDPENDS @@ -352,14 +342,11 @@ NOTE: .bbappend files can impact the dependencies. sorted_rdeps[k2] = 1 all_rdeps = sorted_rdeps.keys() for rdep in all_rdeps: - all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep) + all_p, best = self.tinfoil.get_runtime_providers(rdep) if all_p: if f in all_p: # The recipe provides this one itself, ignore continue - best = bb.providers.filterProvidersRunTime(all_p, rdep, - self.tinfoil.config_data, - self.tinfoil.cooker_data)[0][0] self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) # The RRECOMMENDS @@ -372,14 +359,11 @@ NOTE: .bbappend files can impact the dependencies. sorted_rrecs[k2] = 1 all_rrecs = sorted_rrecs.keys() for rrec in all_rrecs: - all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec) + all_p, best = self.tinfoil.get_runtime_providers(rrec) if all_p: if f in all_p: # The recipe provides this one itself, ignore continue - best = bb.providers.filterProvidersRunTime(all_p, rrec, - self.tinfoil.config_data, - self.tinfoil.cooker_data)[0][0] self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) # The inherit class |