meta/classes/toaster.bbclass


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
}
See http://bugs.openembedded.net/show_bug.cgi?id=2168
Fix for gcc 4.2

--- perl-5.8.8/makedepend.SH.ark	2006-11-01 16:32:05.000000000 +0100
+++ perl-5.8.8/makedepend.SH	2006-11-01 16:32:15.000000000 +0100
@@ -167,6 +167,7 @@
             -e '/^#.*<builtin>/d' \
             -e '/^#.*<built-in>/d' \
             -e '/^#.*<command line>/d' \
+            -e '/^#.*<command-line>/d' \
 	    -e '/^#.*"-"/d' \
 	    -e '/: file path prefix .* never used$/d' \
 	    -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \
#
# Toaster helper class
#
# Copyright (C) 2013 Intel Corporation
#
# Released under the MIT license (see COPYING.MIT)
#
# This bbclass is designed to extract data used by OE-Core during the build process,
# for recording in the Toaster system.
# The data access is synchronous, preserving the build data integrity across
# different builds.
#
# The data is transferred through the event system, using the MetadataEvent objects.
#
# The model is to enable the datadump functions as postfuncs, and have the dump
# executed after the real taskfunc has been executed. This prevents task signature changing
# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled.
#
# To enable, use INHERIT in local.conf:
#
#       INHERIT += "toaster"
#
#
#
#

# Find and dump layer info when we got the layers parsed


python toaster_layerinfo_dumpdata() {
    import subprocess

    def _get_git_branch(layer_path):
        branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
        branch = branch.decode('utf-8')
        branch = branch.replace('refs/heads/', '').rstrip()
        return branch

    def _get_git_revision(layer_path):
        revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
        return revision

    def _get_url_map_name(layer_name):
        """ Some layers have a different name on openembedded.org site,
            this method returns the correct name to use in the URL
        """

        url_name = layer_name
        url_mapping = {'meta': 'openembedded-core'}

        for key in url_mapping.keys():
            if key == layer_name:
                url_name = url_mapping[key]

        return url_name

    def _get_layer_version_information(layer_path):

        layer_version_info = {}
        layer_version_info['branch'] = _get_git_branch(layer_path)
        layer_version_info['commit'] = _get_git_revision(layer_path)
        layer_version_info['priority'] = 0

        return layer_version_info


    def _get_layer_dict(layer_path):

        layer_info = {}
        layer_name = layer_path.split('/')[-1]
        layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
        layer_url_name = _get_url_map_name(layer_name)

        layer_info['name'] = layer_url_name
        layer_info['local_path'] = layer_path
        layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
        layer_info['version'] = _get_layer_version_information(layer_path)

        return layer_info


    bblayers = e.data.getVar("BBLAYERS", True)

    llayerinfo = {}

    for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
        llayerinfo[layer] = _get_layer_dict(layer)


    bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
}

# Dump package file info data

def _toaster_load_pkgdatafile(dirpath, filepath):
    import json
    import re
    pkgdata = {}
    with open(os.path.join(dirpath, filepath), "r") as fin:
        for line in fin:
            try:
                kn, kv = line.strip().split(": ", 1)
                m = re.match(r"^PKG_([^A-Z:]*)", kn)
                if m:
                    pkgdata['OPKGN'] = m.group(1)
                kn = "_".join([x for x in kn.split("_") if x.isupper()])
                pkgdata[kn] = kv.strip()
                if kn == 'FILES_INFO':
                    pkgdata[kn] = json.loads(kv)

            except ValueError:
                pass    # ignore lines without valid key: value pairs
    return pkgdata

python toaster_package_dumpdata() {
    """
    Dumps the data about the packages created by a recipe
    """

    # No need to try and dumpdata if the recipe isn't generating packages
    if not d.getVar('PACKAGES', True):
        return

    pkgdatadir = d.getVar('PKGDESTWORK', True)
    lpkgdata = {}
    datadir = os.path.join(pkgdatadir, 'runtime')

    # scan and send data for each generated package
    for datafile in os.listdir(datadir):
        if not datafile.endswith('.packaged'):
            lpkgdata = _toaster_load_pkgdatafile(datadir, datafile)
            # Fire an event containing the pkg data
            bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
}

# 2. Dump output image files information

python toaster_artifact_dumpdata() {
    """
    Dump data about SDK variables
    """

    event_data = {
      "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME", True)
    }

    bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d)
}

# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data

python toaster_collect_task_stats() {
    import bb.build
    import bb.event
    import bb.data
    import bb.utils
    import os

    toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")

    if not e.data.getVar('BUILDSTATS_BASE', True):
        return  # if we don't have buildstats, we cannot collect stats

    def stat_to_float(value):
        return float(value.strip('% \n\r'))

    def _append_read_list(v):
        lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)

        with open(toaster_statlist_file, "a") as fout:
            taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
            fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))

        bb.utils.unlockfile(lock)

    def _read_stats(filename):
        # seconds
        cpu_time_user = 0
        cpu_time_system = 0

        # bytes
        disk_io_read = 0
        disk_io_write = 0

        started = 0
        ended = 0

        taskname = ''

        statinfo = {}

        with open(filename, 'r') as task_bs:
            for line in task_bs.readlines():
                k,v = line.strip().split(": ", 1)
                statinfo[k] = v

        if "Started" in statinfo:
            started = stat_to_float(statinfo["Started"])

        if "Ended" in statinfo:
            ended = stat_to_float(statinfo["Ended"])

        if "Child rusage ru_utime" in statinfo:
            cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])

        if "Child rusage ru_stime" in statinfo:
            cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])

        if "IO write_bytes" in statinfo:
            write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
            disk_io_write = disk_io_write + write_bytes

        if "IO read_bytes" in statinfo:
            read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
            disk_io_read = disk_io_read + read_bytes

        return {
            'stat_file': filename,
            'cpu_time_user': cpu_time_user,
            'cpu_time_system': cpu_time_system,
            'disk_io_read': disk_io_read,
            'disk_io_write': disk_io_write,
            'started': started,
            'ended': ended
        }

    if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
        _append_read_list(e)
        pass

    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
        events = []
        with open(toaster_statlist_file, "r") as fin:
            for line in fin:
                (taskfile, taskname, filename, recipename) = line.strip().split("::")
                stats = _read_stats(filename)
                events.append((taskfile, taskname, stats, recipename))
        bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
        os.unlink(toaster_statlist_file)
}

# dump relevant build history data as an event when the build is completed

python toaster_buildhistory_dump() {
    import re
    BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
    BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
    pkgdata_dir = e.data.getVar("PKGDATA_DIR", True)


    # scan the build targets for this build
    images = {}
    allpkgs = {}
    files = {}
    for target in e._pkgs:
        target = target.split(':')[0] # strip ':<task>' suffix from the target
        installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
        if os.path.exists(installed_img_path):
            images[target] = {}
            files[target] = {}
            files[target]['dirs'] = []
            files[target]['syms'] = []
            files[target]['files'] = []
            with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
                for line in fin:
                    line = line.rstrip(";")
                    psize, px = line.split("\t")
                    punit, pname = px.split(" ")
                    # this size is "installed-size" as it measures how much space it takes on disk
                    images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}

            with open("%s/depends.dot" % installed_img_path, "r") as fin:
                p = re.compile(r' -> ')
                dot = re.compile(r'.*style=dotted')
                for line in fin:
                    line = line.rstrip(';')
                    linesplit = p.split(line)
                    if len(linesplit) == 2:
                        pname = linesplit[0].rstrip('"').strip('"')
                        dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
                        deptype = "depends"
                        if dot.match(line):
                            deptype = "recommends"
                        if not pname in images[target]:
                            images[target][pname] = {'size': 0, 'depends' : []}
                        if not dependsname in images[target]:
                            images[target][dependsname] = {'size': 0, 'depends' : []}
                        images[target][pname]['depends'].append((dependsname, deptype))

            # files-in-image.txt is only generated if an image file is created,
            # so the file entries ('syms', 'dirs', 'files') for a target will be
            # empty for rootfs builds and other "image" tasks which don't
            # produce image files
            # (e.g. "bitbake core-image-minimal -c populate_sdk")
            files_in_image_path = "%s/files-in-image.txt" % installed_img_path
            if os.path.exists(files_in_image_path):
                with open(files_in_image_path, "r") as fin:
                    for line in fin:
                        lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
                        if lc[0].startswith("l"):
                            files[target]['syms'].append(lc)
                        elif lc[0].startswith("d"):
                            files[target]['dirs'].append(lc)
                        else:
                            files[target]['files'].append(lc)

            for pname in images[target]:
                if not pname in allpkgs:
                    try:
                        pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
                    except IOError as err:
                        if err.errno == 2:
                            # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime
                            continue
                        else:
                            raise
                    allpkgs[pname] = pkgdata


    data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files }

    bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)

}

# set event handlers
addhandler toaster_layerinfo_dumpdata
toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"

addhandler toaster_collect_task_stats
toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"

addhandler toaster_buildhistory_dump
toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"

do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata "
do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata "

do_package[postfuncs] += "toaster_package_dumpdata "
do_package[vardepsexclude] += "toaster_package_dumpdata "

do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "

do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata "
do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata "