From 1befb4a783bb7b7b387d4b5ee08830d9516f1ac2 Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Tue, 9 Dec 2014 11:57:38 +0000 Subject: add option to write offline event log file This patch adds a "-w/--write-log" option to bitbake that writes an event log file for the current build. The name of the file is passed as a parameter to the "-w" argument. If the parameter is the empty string '', the file name is generated in the form bitbake_eventlog_DATE.json, where DATE is the current date and time, with second precision. The "-w" option can also be supplied as the BBEVENTLOG environment variable. We add a script, toater-eventreplay, that reads an event log file and loads the data into a Toaster database, creating a build entry. We modify the toasterui to fix minor issues with reading events from an event log file. Performance impact is undetectable under no-task executed builds. Signed-off-by: Alexandru DAMIAN --- bin/bitbake | 12 +++- bin/toaster-eventreplay | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100755 bin/toaster-eventreplay (limited to 'bin') diff --git a/bin/bitbake b/bin/bitbake index 7f8449c7b..d46c3dde3 100755 --- a/bin/bitbake +++ b/bin/bitbake @@ -196,6 +196,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): parser.add_option("", "--status-only", help = "Check the status of the remote bitbake server.", action = "store_true", dest = "status_only", default = False) + parser.add_option("-w", "--write-log", help = "Writes the event log of the build to a bitbake event json file. Use '' (empty string) to assign the name automatically.", + action = "store", dest = "writeeventlog") + options, targets = parser.parse_args(sys.argv) # some environmental variables set also configuration options @@ -206,6 +209,14 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): if "BBTOKEN" in os.environ: options.xmlrpctoken = os.environ["BBTOKEN"] + if "BBEVENTLOG" is os.environ: + options.writeeventlog = os.environ["BBEVENTLOG"] + + # fill in proper log name if not supplied + if options.writeeventlog is not None and len(options.writeeventlog) == 0: + import datetime + options.writeeventlog = "bitbake_eventlog_%s.json" % datetime.datetime.now().strftime("%Y%m%d%H%M%S") + # if BBSERVER says to autodetect, let's do that if options.remote_server: [host, port] = options.remote_server.split(":", 2) @@ -266,7 +277,6 @@ def start_server(servermodule, configParams, configuration, features): return server - def main(): configParams = BitBakeConfigParameters() diff --git a/bin/toaster-eventreplay b/bin/toaster-eventreplay new file mode 100755 index 000000000..624829aea --- /dev/null +++ b/bin/toaster-eventreplay @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2014 Alex Damian +# +# This file re-uses code spread throughout other Bitbake source files. +# As such, all other copyrights belong to their own right holders. +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# This command takes a filename as a single parameter. The filename is read +# as a build eventlog, and the ToasterUI is used to process events in the file +# and log data in the database + +import os +import sys, logging + +# mangle syspath to allow easy import of modules +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'lib')) + + +import bb.cooker +from bb.ui import toasterui +import sys +import logging + +logger = logging.getLogger(__name__) +console = logging.StreamHandler(sys.stdout) +format_str = "%(levelname)s: %(message)s" +logging.basicConfig(format=format_str) + + +import json, pickle + + +class FileReadEventsServerConnection(): + """ Emulates a connection to a bitbake server that feeds + events coming actually read from a saved log file. + """ + + class MockConnection(): + """ fill-in for the proxy to the server. we just return generic data + """ + def __init__(self, sc): + self._sc = sc + + def runCommand(self, commandArray): + """ emulates running a command on the server; only read-only commands are accepted """ + command_name = commandArray[0] + + if command_name == "getVariable": + if commandArray[1] in self._sc._variables: + return (self._sc._variables[commandArray[1]]['v'], None) + return (None, "Missing variable") + + elif command_name == "getAllKeysWithFlags": + dump = {} + flaglist = commandArray[1] + for k in self._sc._variables.keys(): + try: + if not k.startswith("__"): + v = self._sc._variables[k]['v'] + dump[k] = { + 'v' : v , + 'history' : self._sc._variables[k]['history'], + } + for d in flaglist: + dump[k][d] = self._sc._variables[k][d] + except Exception as e: + print(e) + return (dump, None) + else: + raise Exception("Command %s not implemented" % commandArray[0]) + + def terminateServer(self): + """ do not do anything """ + pass + + + + class EventReader(): + def __init__(self, sc): + self._sc = sc + self.firstraise = 0 + + def _create_event(self, line): + def _import_class(name): + assert len(name) > 0 + assert "." in name, name + + components = name.strip().split(".") + modulename = ".".join(components[:-1]) + moduleklass = components[-1] + + module = __import__(modulename, fromlist=[str(moduleklass)]) + return getattr(module, moduleklass) + + # we build a toaster event out of current event log line + try: + event_data = json.loads(line.strip()) + event_class = _import_class(event_data['class']) + event_object = pickle.loads(json.loads(event_data['vars'])) + except ValueError as e: + print("Failed loading ", line) + raise e + + if not isinstance(event_object, event_class): + raise Exception("Error loading objects %s class %s ", event_object, event_class) + + return event_object + + def waitEvent(self, timeout): + + nextline = self._sc._eventfile.readline() + if len(nextline) == 0: + # the build data ended, while toasterui still waits for events. + # this happens when the server was abruptly stopped, so we simulate this + self.firstraise += 1 + if self.firstraise == 1: + raise KeyboardInterrupt() + else: + return None + else: + self._sc.lineno += 1 + return self._create_event(nextline) + + + def _readVariables(self, variableline): + self._variables = json.loads(variableline.strip())['allvariables'] + + + def __init__(self, file_name): + self.connection = FileReadEventsServerConnection.MockConnection(self) + self._eventfile = open(file_name, "r") + + # we expect to have the variable dump at the start of the file + self.lineno = 1 + self._readVariables(self._eventfile.readline()) + + self.events = FileReadEventsServerConnection.EventReader(self) + + + + + +class MockConfigParameters(): + """ stand-in for cookerdata.ConfigParameters; as we don't really config a cooker, this + serves just to supply needed interfaces for the toaster ui to work """ + def __init__(self): + self.observe_only = True # we can only read files + + +# run toaster ui on our mock bitbake class +if __name__ == "__main__": + if len(sys.argv) < 2: + logger.error("Usage: %s event.log " % sys.argv[0]) + sys.exit(1) + + file_name = sys.argv[-1] + mock_connection = FileReadEventsServerConnection(file_name) + configParams = MockConfigParameters() + + # run the main program + toasterui.main(mock_connection.connection, mock_connection.events, configParams) -- cgit 1.2.3-korg