aboutsummaryrefslogtreecommitdiffstats
path: root/meta/classes/toaster.bbclass
blob: f365c091420076ca10328f46df3445c72468822d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
#
# Toaster helper class
#
# Copyright (C) 2013 Intel Corporation
#
# Released under the MIT license (see COPYING.MIT)
#
# This bbclass is designed to extract data used by OE-Core during the build process,
# for recording in the Toaster system.
# The data access is synchronous, preserving the build data integrity across
# different builds.
#
# The data is transferred through the event system, using the MetadataEvent objects.
#
# The model is to enable the datadump functions as postfuncs, and have the dump
# executed after the real taskfunc has been executed. This prevents task signature changing
# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled.
#
# To enable, use INHERIT in local.conf:
#
#       INHERIT += "toaster"
#
#
#
#

# Find and dump layer info when we got the layers parsed



python toaster_layerinfo_dumpdata() {
    import subprocess

    def _get_git_branch(layer_path):
        branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
        branch = branch.decode('utf-8')
        branch = branch.replace('refs/heads/', '').rstrip()
        return branch

    def _get_git_revision(layer_path):
        revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
        return revision

    def _get_url_map_name(layer_name):
        """ Some layers have a different name on openembedded.org site,
            this method returns the correct name to use in the URL
        """

        url_name = layer_name
        url_mapping = {'meta': 'openembedded-core'}

        for key in url_mapping.keys():
            if key == layer_name:
                url_name = url_mapping[key]

        return url_name

    def _get_layer_version_information(layer_path):

        layer_version_info = {}
        layer_version_info['branch'] = _get_git_branch(layer_path)
        layer_version_info['commit'] = _get_git_revision(layer_path)
        layer_version_info['priority'] = 0

        return layer_version_info


    def _get_layer_dict(layer_path):

        layer_info = {}
        layer_name = layer_path.split('/')[-1]
        layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
        layer_url_name = _get_url_map_name(layer_name)

        layer_info['name'] = layer_url_name
        layer_info['local_path'] = layer_path
        layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
        layer_info['version'] = _get_layer_version_information(layer_path)

        return layer_info


    bblayers = e.data.getVar("BBLAYERS")

    llayerinfo = {}

    for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
        llayerinfo[layer] = _get_layer_dict(layer)


    bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
}

# Dump package file info data

def _toaster_load_pkgdatafile(dirpath, filepath):
    import json
    import re
    pkgdata = {}
    with open(os.path.join(dirpath, filepath), "r") as fin:
        for line in fin:
            try:
                kn, kv = line.strip().split(": ", 1)
                m = re.match(r"^PKG:([^A-Z:]*)", kn)
                if m:
                    pkgdata['OPKGN'] = m.group(1)
                kn = kn.split(":")[0]
                pkgdata[kn] = kv
                if kn.startswith('FILES_INFO'):
                    pkgdata[kn] = json.loads(kv)

            except ValueError:
                pass    # ignore lines without valid key: value pairs
    return pkgdata

def _toaster_dumpdata(pkgdatadir, d):
    """
    Dumps the data about the packages created by a recipe
    """

    # No need to try and dumpdata if the recipe isn't generating packages
    if not d.getVar('PACKAGES'):
        return

    lpkgdata = {}
    datadir = os.path.join(pkgdatadir, 'runtime')

    # scan and send data for each generated package
    if os.path.exists(datadir):
        for datafile in os.listdir(datadir):
            if not datafile.endswith('.packaged'):
                lpkgdata = _toaster_load_pkgdatafile(datadir, datafile)
                # Fire an event containing the pkg data
                bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)

python toaster_package_dumpdata() {
    _toaster_dumpdata(d.getVar('PKGDESTWORK'), d)
}

python toaster_packagedata_dumpdata() {
    # This path needs to match do_packagedata[sstate-inputdirs]
    _toaster_dumpdata(os.path.join(d.getVar('WORKDIR'), 'pkgdata-pdata-input'), d)
}

# 2. Dump output image files information

python toaster_artifact_dumpdata() {
    """
    Dump data about SDK variables
    """

    event_data = {
      "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME")
    }

    bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d)
}

# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data

python toaster_collect_task_stats() {
    import bb.build
    import bb.event
    import bb.data
    import bb.utils
    import os

    if not e.data.getVar('BUILDSTATS_BASE'):
        return  # if we don't have buildstats, we cannot collect stats

    toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE'), "toasterstatlist")

    def stat_to_float(value):
        return float(value.strip('% \n\r'))

    def _append_read_list(v):
        lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)

        with open(toaster_statlist_file, "a") as fout:
            taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
            fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))

        bb.utils.unlockfile(lock)

    def _read_stats(filename):
        # seconds
        cpu_time_user = 0
        cpu_time_system = 0

        # bytes
        disk_io_read = 0
        disk_io_write = 0

        started = 0
        ended = 0

        taskname = ''

        statinfo = {}

        with open(filename, 'r') as task_bs:
            for line in task_bs.readlines():
                k,v = line.strip().split(": ", 1)
                statinfo[k] = v

        if "Started" in statinfo:
            started = stat_to_float(statinfo["Started"])

        if "Ended" in statinfo:
            ended = stat_to_float(statinfo["Ended"])

        if "Child rusage ru_utime" in statinfo:
            cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])

        if "Child rusage ru_stime" in statinfo:
            cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])

        if "IO write_bytes" in statinfo:
            write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
            disk_io_write = disk_io_write + write_bytes

        if "IO read_bytes" in statinfo:
            read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
            disk_io_read = disk_io_read + read_bytes

        return {
            'stat_file': filename,
            'cpu_time_user': cpu_time_user,
            'cpu_time_system': cpu_time_system,
            'disk_io_read': disk_io_read,
            'disk_io_write': disk_io_write,
            'started': started,
            'ended': ended
        }

    if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
        _append_read_list(e)
        pass

    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
        events = []
        with open(toaster_statlist_file, "r") as fin:
            for line in fin:
                (taskfile, taskname, filename, recipename) = line.strip().split("::")
                stats = _read_stats(filename)
                events.append((taskfile, taskname, stats, recipename))
        bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
        os.unlink(toaster_statlist_file)
}

# dump relevant build history data as an event when the build is completed

python toaster_buildhistory_dump() {
    import re
    BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
    BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
    pkgdata_dir = e.data.getVar("PKGDATA_DIR")


    # scan the build targets for this build
    images = {}
    allpkgs = {}
    files = {}
    for target in e._pkgs:
        target = target.split(':')[0] # strip ':<task>' suffix from the target
        installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
        if os.path.exists(installed_img_path):
            images[target] = {}
            files[target] = {}
            files[target]['dirs'] = []
            files[target]['syms'] = []
            files[target]['files'] = []
            with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
                for line in fin:
                    line = line.rstrip(";")
                    psize, punit, pname = line.split()
                    # this size is "installed-size" as it measures how much space it takes on disk
                    images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}

            with open("%s/depends.dot" % installed_img_path, "r") as fin:
                p = re.compile(r'\s*"(?P<name>[^"]+)"\s*->\s*"(?P<dep>[^"]+)"(?P<rec>.*?\[style=dotted\])?')
                for line in fin:
                    m = p.match(line)
                    if not m:
                        continue
                    pname = m.group('name')
                    dependsname = m.group('dep')
                    deptype = 'recommends' if m.group('rec') else 'depends'

                    # If RPM is used for packaging, then there may be
                    # dependencies such as "/bin/sh", which will confuse
                    # _toaster_load_pkgdatafile() later on. While at it, ignore
                    # any dependencies that contain parentheses, e.g.,
                    # "libc.so.6(GLIBC_2.7)".
                    if dependsname.startswith('/') or '(' in dependsname:
                        continue

                    if not pname in images[target]:
                        images[target][pname] = {'size': 0, 'depends' : []}
                    if not dependsname in images[target]:
                        images[target][dependsname] = {'size': 0, 'depends' : []}
                    images[target][pname]['depends'].append((dependsname, deptype))

            # files-in-image.txt is only generated if an image file is created,
            # so the file entries ('syms', 'dirs', 'files') for a target will be
            # empty for rootfs builds and other "image" tasks which don't
            # produce image files
            # (e.g. "bitbake core-image-minimal -c populate_sdk")
            files_in_image_path = "%s/files-in-image.txt" % installed_img_path
            if os.path.exists(files_in_image_path):
                with open(files_in_image_path, "r") as fin:
                    for line in fin:
                        lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
                        if lc[0].startswith("l"):
                            files[target]['syms'].append(lc)
                        elif lc[0].startswith("d"):
                            files[target]['dirs'].append(lc)
                        else:
                            files[target]['files'].append(lc)

            for pname in images[target]:
                if not pname in allpkgs:
                    try:
                        pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
                    except IOError as err:
                        if err.errno == 2:
                            # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime
                            continue
                        else:
                            raise
                    allpkgs[pname] = pkgdata


    data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files }

    bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)

}

# get list of artifacts from sstate manifest
python toaster_artifacts() {
    if e.taskname in ["do_deploy", "do_image_complete", "do_populate_sdk", "do_populate_sdk_ext"]:
        d2 = d.createCopy()
        d2.setVar('FILE', e.taskfile)
        # Use 'stamp-extra-info' if present, else use workaround
        # to determine 'SSTATE_MANMACH'
        extrainf = d2.getVarFlag(e.taskname, 'stamp-extra-info')
        if extrainf:
            d2.setVar('SSTATE_MANMACH', extrainf)
        else:
            if "do_populate_sdk" == e.taskname:
                d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}${SDKMACHINE}"))
            else:
                d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}"))
        manifest = oe.sstatesig.sstate_get_manifest_filename(e.taskname[3:], d2)[0]

        if os.access(manifest, os.R_OK):
            with open(manifest) as fmanifest:
                artifacts = [fname.strip() for fname in fmanifest]
                data = {"task": e.taskid, "artifacts": artifacts}
                bb.event.fire(bb.event.MetadataEvent("TaskArtifacts", data), d2)
}

# set event handlers
addhandler toaster_layerinfo_dumpdata
toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"

addhandler toaster_collect_task_stats
toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"

addhandler toaster_buildhistory_dump
toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"

addhandler toaster_artifacts
toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted"

do_packagedata_setscene[postfuncs] += "toaster_packagedata_dumpdata "
do_packagedata_setscene[vardepsexclude] += "toaster_packagedata_dumpdata "

do_package[postfuncs] += "toaster_package_dumpdata "
do_package[vardepsexclude] += "toaster_package_dumpdata "

#do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
#do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "

#do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata "
#do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata "