1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
|
# Class to avoid copying packages into the feed if they haven't materially changed
#
# Copyright (C) 2015 Intel Corporation
# Released under the MIT license (see COPYING.MIT for details)
#
# This class effectively intercepts packages as they are written out by
# do_package_write_*, causing them to be written into a different
# directory where we can compare them to whatever older packages might
# be in the "real" package feed directory, and avoid copying the new
# package to the feed if it has not materially changed. The idea is to
# avoid unnecessary churn in the packages when dependencies trigger task
# reexecution (and thus repackaging). Enabling the class is simple:
#
# INHERIT += "packagefeed-stability"
#
# Caveats:
# 1) Latest PR values in the build system may not match those in packages
# seen on the target (naturally)
# 2) If you rebuild from sstate without the existing package feed present,
# you will lose the "state" of the package feed i.e. the preserved old
# package versions. Not the end of the world, but would negate the
# entire purpose of this class.
#
# Note that running -c cleanall on a recipe will purposely delete the old
# package files so they will definitely be copied the next time.
python() {
# Package backend agnostic intercept
# This assumes that the package_write task is called package_write_<pkgtype>
# and that the directory in which packages should be written is
# pointed to by the variable DEPLOY_DIR_<PKGTYPE>
for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
if pkgclass.startswith('package_'):
pkgtype = pkgclass.split('_', 1)[1]
pkgwritefunc = 'do_package_write_%s' % pkgtype
sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
deploydirvarref = '${' + deploydirvar + '}'
pkgcomparefunc = 'do_package_compare_%s' % pkgtype
if bb.data.inherits_class('image', d):
d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)
if bb.data.inherits_class('populate_sdk_base', d):
d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)
if bb.data.inherits_class('populate_sdk_ext', d):
d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)
d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)
if d.getVarFlag(pkgwritefunc, 'noexec', True) or (not d.getVarFlag(pkgwritefunc, 'task', True)) or pkgwritefunc in (d.getVar('__BBDELTASKS', True) or []):
# Packaging is disabled for this recipe, we shouldn't do anything
continue
if deploydirvarref in sstate_outputdirs:
# Set intermediate output directory
d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff'))
d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
}
# This isn't the real task function - it's a template that we use in the
# anonymous python code above
fakeroot python do_package_compare () {
currenttask = d.getVar('BB_CURRENTTASK', True)
pkgtype = currenttask.rsplit('_', 1)[1]
package_compare_impl(pkgtype, d)
}
def package_compare_impl(pkgtype, d):
import errno
import fnmatch
import glob
import subprocess
import oe.sstatesig
pn = d.getVar('PN', True)
deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
prepath = deploydir + '-prediff/'
# Find out PKGR values are
pkgdatadir = d.getVar('PKGDATA_DIR', True)
packages = []
try:
with open(os.path.join(pkgdatadir, pn), 'r') as f:
for line in f:
if line.startswith('PACKAGES:'):
packages = line.split(':', 1)[1].split()
break
except IOError as e:
if e.errno == errno.ENOENT:
pass
if not packages:
bb.debug(2, '%s: no packages, nothing to do' % pn)
return
pkgrvalues = {}
rpkgnames = {}
rdepends = {}
pkgvvalues = {}
for pkg in packages:
with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
for line in f:
if line.startswith('PKGR:'):
pkgrvalues[pkg] = line.split(':', 1)[1].strip()
if line.startswith('PKGV:'):
pkgvvalues[pkg] = line.split(':', 1)[1].strip()
elif line.startswith('PKG_%s:' % pkg):
rpkgnames[pkg] = line.split(':', 1)[1].strip()
elif line.startswith('RDEPENDS_%s:' % pkg):
rdepends[pkg] = line.split(':', 1)[1].strip()
# Prepare a list of the runtime package names for packages that were
# actually produced
rpkglist = []
for pkg, rpkg in rpkgnames.iteritems():
if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
rpkglist.append((rpkg, pkg))
rpkglist.sort(key=lambda x: len(x[0]), reverse=True)
pvu = d.getVar('PV', False)
if '$' + '{SRCPV}' in pvu:
pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
else:
pvprefix = None
pkgwritetask = 'package_write_%s' % pkgtype
files = []
copypkgs = []
manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
with open(manifest, 'r') as f:
for line in f:
if line.startswith(prepath):
srcpath = line.rstrip()
if os.path.isfile(srcpath):
destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))
# This is crude but should work assuming the output
# package file name starts with the package name
# and rpkglist is sorted by length (descending)
pkgbasename = os.path.basename(destpath)
pkgname = None
for rpkg, pkg in rpkglist:
if pkgbasename.startswith(rpkg):
pkgr = pkgrvalues[pkg]
destpathspec = destpath.replace(pkgr, '*')
if pvprefix:
pkgv = pkgvvalues[pkg]
if pkgv.startswith(pvprefix):
pkgvsuffix = pkgv[len(pvprefix):]
if '+' in pkgvsuffix:
newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
destpathspec = destpathspec.replace(pkgv, newpkgv)
pkgname = pkg
break
else:
bb.warn('Unable to map %s back to package' % pkgbasename)
destpathspec = destpath
oldfiles = glob.glob(destpathspec)
oldfile = None
docopy = True
if oldfiles:
oldfile = oldfiles[-1]
result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
if result == 0:
docopy = False
files.append((pkgname, pkgbasename, srcpath, oldfile, destpath))
bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy))
if docopy:
copypkgs.append(pkgname)
# Ensure that dependencies on specific versions (such as -dev on the
# main package) are copied in lock-step
changed = True
while changed:
rpkgdict = {x[0]: x[1] for x in rpkglist}
changed = False
for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
rdeps = rdepends.get(pkgname, None)
if not rdeps:
continue
rdepvers = bb.utils.explode_dep_versions2(rdeps)
for rdep, versions in rdepvers.iteritems():
dep = rpkgdict.get(rdep, None)
for version in versions:
if version and version.startswith('= '):
if dep in copypkgs and not pkgname in copypkgs:
bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep))
changed = True
copypkgs.append(pkgname)
elif pkgname in copypkgs and not dep in copypkgs:
bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname))
changed = True
copypkgs.append(dep)
# Read in old manifest so we can delete any packages we aren't going to replace or preserve
pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
try:
with open(pcmanifest, 'r') as f:
knownfiles = [x[3] for x in files if x[3]]
for line in f:
fn = line.rstrip()
if fn:
if fn in knownfiles:
knownfiles.remove(fn)
else:
try:
os.remove(fn)
bb.warn('Removed old package %s' % fn)
except OSError as e:
if e.errno == errno.ENOENT:
pass
except IOError as e:
if e.errno == errno.ENOENT:
pass
# Create new manifest
with open(pcmanifest, 'w') as f:
for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
if pkgname in copypkgs:
bb.warn('Copying %s' % pkgbasename)
destdir = os.path.dirname(destpath)
bb.utils.mkdirhier(destdir)
if oldfile:
try:
os.remove(oldfile)
except OSError as e:
if e.errno == errno.ENOENT:
pass
if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
# Use a hard link to save space
os.link(srcpath, destpath)
else:
shutil.copyfile(srcpath, destpath)
f.write('%s\n' % destpath)
else:
bb.warn('Not copying %s' % pkgbasename)
f.write('%s\n' % oldfile)
do_cleanall_append() {
import errno
for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
if pkgclass.startswith('package_'):
pkgtype = pkgclass.split('_', 1)[1]
deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
prepath = deploydir + '-prediff'
pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
try:
with open(pcmanifest, 'r') as f:
for line in f:
fn = line.rstrip()
if fn:
try:
os.remove(fn)
except OSError as e:
if e.errno == errno.ENOENT:
pass
os.remove(pcmanifest)
except IOError as e:
if e.errno == errno.ENOENT:
pass
}
|
