diff options
Diffstat (limited to 'scripts/lib/mic/plugins')
| -rw-r--r-- | scripts/lib/mic/plugins/backend/yumpkgmgr.py | 490 | ||||
| -rwxr-xr-x | scripts/lib/mic/plugins/backend/zypppkgmgr.py | 973 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/hook/.py | 0 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/hook/empty_hook.py | 3 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/imager/fs_plugin.py | 143 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/imager/livecd_plugin.py | 255 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/imager/liveusb_plugin.py | 260 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/imager/loop_plugin.py | 255 | ||||
| -rw-r--r-- | scripts/lib/mic/plugins/imager/raw_plugin.py | 275 |
9 files changed, 2654 insertions, 0 deletions
diff --git a/scripts/lib/mic/plugins/backend/yumpkgmgr.py b/scripts/lib/mic/plugins/backend/yumpkgmgr.py new file mode 100644 index 0000000000..955f813109 --- /dev/null +++ b/scripts/lib/mic/plugins/backend/yumpkgmgr.py @@ -0,0 +1,490 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2007 Red Hat Inc. +# Copyright (c) 2010, 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os, sys +import re +import tempfile +import glob +from string import Template + +import rpmUtils +import yum + +from mic import msger +from mic.kickstart import ksparser +from mic.utils import misc, rpmmisc +from mic.utils.grabber import TextProgress +from mic.utils.proxy import get_proxy_for +from mic.utils.errors import CreatorError +from mic.imager.baseimager import BaseImageCreator + +YUMCONF_TEMP = """[main] +installroot=$installroot +cachedir=/var/cache/yum +persistdir=/var/lib/yum +plugins=0 +reposdir= +failovermethod=priority +http_caching=packages +sslverify=1 +""" + +class MyYumRepository(yum.yumRepo.YumRepository): + def __del__(self): + pass + + def dirSetup(self): + super(MyYumRepository, self).dirSetup() + # relocate package dir + pkgdir = os.path.join(self.basecachedir, 'packages', self.id) + self.setAttribute('_dir_setup_pkgdir', pkgdir) + self._dirSetupMkdir_p(self.pkgdir) + + def _getFile(self, url=None, + relative=None, + local=None, + start=None, + end=None, + copy_local=None, + checkfunc=None, + text=None, + reget='simple', + cache=True, + size=None): + + m2c_connection = None + if not self.sslverify: + try: + import M2Crypto + m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck + M2Crypto.SSL.Connection.clientPostConnectionCheck = None + except ImportError, err: + raise CreatorError("%s, please try to install python-m2crypto" % str(err)) + + proxy = None + if url: + proxy = get_proxy_for(url) + else: + proxy = get_proxy_for(self.urls[0]) + + if proxy: + self.proxy = str(proxy) + + size = int(size) if size else None + rvalue = super(MyYumRepository, self)._getFile(url, + relative, + local, + start, + end, + copy_local, + checkfunc, + text, + reget, + cache, + size) + + if m2c_connection and \ + not M2Crypto.SSL.Connection.clientPostConnectionCheck: + M2Crypto.SSL.Connection.clientPostConnectionCheck = m2c_connection + + return rvalue + +from mic.pluginbase import BackendPlugin +class Yum(BackendPlugin, yum.YumBase): + name = 'yum' + + def __init__(self, target_arch, instroot, cachedir): + yum.YumBase.__init__(self) + + self.cachedir = cachedir + self.instroot = instroot + self.target_arch = target_arch + + if self.target_arch: + if not rpmUtils.arch.arches.has_key(self.target_arch): + rpmUtils.arch.arches["armv7hl"] = "noarch" + rpmUtils.arch.arches["armv7tnhl"] = "armv7nhl" + rpmUtils.arch.arches["armv7tnhl"] = "armv7thl" + rpmUtils.arch.arches["armv7thl"] = "armv7hl" + rpmUtils.arch.arches["armv7nhl"] = "armv7hl" + self.arch.setup_arch(self.target_arch) + + self.__pkgs_license = {} + self.__pkgs_content = {} + self.__pkgs_vcsinfo = {} + + self.install_debuginfo = False + + def doFileLogSetup(self, uid, logfile): + # don't do the file log for the livecd as it can lead to open fds + # being left and an inability to clean up after ourself + pass + + def close(self): + try: + os.unlink(self.confpath) + os.unlink(self.conf.installroot + "/yum.conf") + except: + pass + + if self.ts: + self.ts.close() + self._delRepos() + self._delSacks() + yum.YumBase.close(self) + self.closeRpmDB() + + if not os.path.exists("/etc/fedora-release") and \ + not os.path.exists("/etc/meego-release"): + for i in range(3, os.sysconf("SC_OPEN_MAX")): + try: + os.close(i) + except: + pass + + def __del__(self): + pass + + def _writeConf(self, confpath, installroot): + conf = Template(YUMCONF_TEMP).safe_substitute(installroot=installroot) + + f = file(confpath, "w+") + f.write(conf) + f.close() + + os.chmod(confpath, 0644) + + def _cleanupRpmdbLocks(self, installroot): + # cleans up temporary files left by bdb so that differing + # versions of rpm don't cause problems + for f in glob.glob(installroot + "/var/lib/rpm/__db*"): + os.unlink(f) + + def setup(self): + # create yum.conf + (fn, self.confpath) = tempfile.mkstemp(dir=self.cachedir, + prefix='yum.conf-') + os.close(fn) + self._writeConf(self.confpath, self.instroot) + self._cleanupRpmdbLocks(self.instroot) + # do setup + self.doConfigSetup(fn = self.confpath, root = self.instroot) + self.conf.cache = 0 + self.doTsSetup() + self.doRpmDBSetup() + self.doRepoSetup() + self.doSackSetup() + + def preInstall(self, pkg): + # FIXME: handle pre-install package + return None + + def selectPackage(self, pkg): + """Select a given package. + Can be specified with name.arch or name* + """ + + try: + self.install(pattern = pkg) + return None + except yum.Errors.InstallError: + return "No package(s) available to install" + except yum.Errors.RepoError, e: + raise CreatorError("Unable to download from repo : %s" % (e,)) + except yum.Errors.YumBaseError, e: + raise CreatorError("Unable to install: %s" % (e,)) + + def deselectPackage(self, pkg): + """Deselect package. Can be specified as name.arch or name* + """ + + sp = pkg.rsplit(".", 2) + txmbrs = [] + if len(sp) == 2: + txmbrs = self.tsInfo.matchNaevr(name=sp[0], arch=sp[1]) + + if len(txmbrs) == 0: + exact, match, unmatch = yum.packages.parsePackages( + self.pkgSack.returnPackages(), + [pkg], + casematch=1) + for p in exact + match: + txmbrs.append(p) + + if len(txmbrs) > 0: + for x in txmbrs: + self.tsInfo.remove(x.pkgtup) + # we also need to remove from the conditionals + # dict so that things don't get pulled back in as a result + # of them. yes, this is ugly. conditionals should die. + for req, pkgs in self.tsInfo.conditionals.iteritems(): + if x in pkgs: + pkgs.remove(x) + self.tsInfo.conditionals[req] = pkgs + else: + msger.warning("No such package %s to remove" %(pkg,)) + + def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT): + try: + yum.YumBase.selectGroup(self, grp) + if include == ksparser.GROUP_REQUIRED: + for p in grp.default_packages.keys(): + self.deselectPackage(p) + + elif include == ksparser.GROUP_ALL: + for p in grp.optional_packages.keys(): + self.selectPackage(p) + + return None + except (yum.Errors.InstallError, yum.Errors.GroupsError), e: + return e + except yum.Errors.RepoError, e: + raise CreatorError("Unable to download from repo : %s" % (e,)) + except yum.Errors.YumBaseError, e: + raise CreatorError("Unable to install: %s" % (e,)) + + def addRepository(self, name, url = None, mirrorlist = None, proxy = None, + proxy_username = None, proxy_password = None, + inc = None, exc = None, ssl_verify=True, nocache=False, + cost = None, priority=None): + # TODO: Handle priority attribute for repos + def _varSubstitute(option): + # takes a variable and substitutes like yum configs do + option = option.replace("$basearch", rpmUtils.arch.getBaseArch()) + option = option.replace("$arch", rpmUtils.arch.getCanonArch()) + return option + + repo = MyYumRepository(name) + + # Set proxy + repo.proxy = proxy + repo.proxy_username = proxy_username + repo.proxy_password = proxy_password + + if url: + repo.baseurl.append(_varSubstitute(url)) + + # check LICENSE files + if not rpmmisc.checkRepositoryEULA(name, repo): + msger.warning('skip repo:%s for failed EULA confirmation' % name) + return None + + if mirrorlist: + repo.mirrorlist = _varSubstitute(mirrorlist) + + conf = yum.config.RepoConf() + for k, v in conf.iteritems(): + if v or not hasattr(repo, k): + repo.setAttribute(k, v) + + repo.sslverify = ssl_verify + repo.cache = not nocache + + repo.basecachedir = self.cachedir + repo.base_persistdir = self.conf.persistdir + repo.failovermethod = "priority" + repo.metadata_expire = 0 + # Enable gpg check for verifying corrupt packages + repo.gpgcheck = 1 + repo.enable() + repo.setup(0) + self.repos.add(repo) + if cost: + repo.cost = cost + + msger.verbose('repo: %s was added' % name) + return repo + + def installLocal(self, pkg, po=None, updateonly=False): + ts = rpmUtils.transaction.initReadOnlyTransaction() + try: + hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg) + except rpmUtils.RpmUtilsError, e: + raise yum.Errors.MiscError, \ + 'Could not open local rpm file: %s: %s' % (pkg, e) + + self.deselectPackage(hdr['name']) + yum.YumBase.installLocal(self, pkg, po, updateonly) + + def installHasFile(self, file): + provides_pkg = self.whatProvides(file, None, None) + dlpkgs = map( + lambda x: x.po, + filter( + lambda txmbr: txmbr.ts_state in ("i", "u"), + self.tsInfo.getMembers())) + + for p in dlpkgs: + for q in provides_pkg: + if (p == q): + return True + + return False + + def runInstall(self, checksize = 0): + os.environ["HOME"] = "/" + os.environ["LD_PRELOAD"] = "" + try: + (res, resmsg) = self.buildTransaction() + except yum.Errors.RepoError, e: + raise CreatorError("Unable to download from repo : %s" %(e,)) + + if res != 2: + raise CreatorError("Failed to build transaction : %s" \ + % str.join("\n", resmsg)) + + dlpkgs = map( + lambda x: x.po, + filter( + lambda txmbr: txmbr.ts_state in ("i", "u"), + self.tsInfo.getMembers())) + + # record all pkg and the content + for pkg in dlpkgs: + pkg_long_name = misc.RPM_FMT % { + 'name': pkg.name, + 'arch': pkg.arch, + 'version': pkg.version, + 'release': pkg.release + } + self.__pkgs_content[pkg_long_name] = pkg.files + license = pkg.license + if license in self.__pkgs_license.keys(): + self.__pkgs_license[license].append(pkg_long_name) + else: + self.__pkgs_license[license] = [pkg_long_name] + + total_count = len(dlpkgs) + cached_count = 0 + download_total_size = sum(map(lambda x: int(x.packagesize), dlpkgs)) + + msger.info("\nChecking packages cached ...") + for po in dlpkgs: + local = po.localPkg() + repo = filter(lambda r: r.id == po.repoid, self.repos.listEnabled())[0] + if not repo.cache and os.path.exists(local): + os.unlink(local) + if not os.path.exists(local): + continue + if not self.verifyPkg(local, po, False): + msger.warning("Package %s is damaged: %s" \ + % (os.path.basename(local), local)) + else: + download_total_size -= int(po.packagesize) + cached_count +=1 + + cache_avail_size = misc.get_filesystem_avail(self.cachedir) + if cache_avail_size < download_total_size: + raise CreatorError("No enough space used for downloading.") + + # record the total size of installed pkgs + pkgs_total_size = 0L + for x in dlpkgs: + if hasattr(x, 'installedsize'): + pkgs_total_size += int(x.installedsize) + else: + pkgs_total_size += int(x.size) + + # check needed size before actually download and install + if checksize and pkgs_total_size > checksize: + raise CreatorError("No enough space used for installing, " + "please resize partition size in ks file") + + msger.info("Packages: %d Total, %d Cached, %d Missed" \ + % (total_count, cached_count, total_count - cached_count)) + + try: + repos = self.repos.listEnabled() + for repo in repos: + repo.setCallback(TextProgress(total_count - cached_count)) + + self.downloadPkgs(dlpkgs) + # FIXME: sigcheck? + + self.initActionTs() + self.populateTs(keepold=0) + + deps = self.ts.check() + if len(deps) != 0: + # This isn't fatal, Ubuntu has this issue but it is ok. + msger.debug(deps) + msger.warning("Dependency check failed!") + + rc = self.ts.order() + if rc != 0: + raise CreatorError("ordering packages for installation failed") + + # FIXME: callback should be refactored a little in yum + cb = rpmmisc.RPMInstallCallback(self.ts) + cb.tsInfo = self.tsInfo + cb.filelog = False + + msger.warning('\nCaution, do NOT interrupt the installation, ' + 'else mic cannot finish the cleanup.') + + installlogfile = "%s/__catched_stderr.buf" % (self.instroot) + msger.enable_logstderr(installlogfile) + self.runTransaction(cb) + self._cleanupRpmdbLocks(self.conf.installroot) + + except rpmUtils.RpmUtilsError, e: + raise CreatorError("mic does NOT support delta rpm: %s" % e) + except yum.Errors.RepoError, e: + raise CreatorError("Unable to download from repo : %s" % e) + except yum.Errors.YumBaseError, e: + raise CreatorError("Unable to install: %s" % e) + finally: + msger.disable_logstderr() + + def getVcsInfo(self): + return self.__pkgs_vcsinfo + + def getAllContent(self): + return self.__pkgs_content + + def getPkgsLicense(self): + return self.__pkgs_license + + def getFilelist(self, pkgname): + if not pkgname: + return None + + pkg = filter(lambda txmbr: txmbr.po.name == pkgname, self.tsInfo.getMembers()) + if not pkg: + return None + return pkg[0].po.filelist + + def package_url(self, pkgname): + pkgs = self.pkgSack.searchNevra(name=pkgname) + if pkgs: + proxy = None + proxies = None + url = pkgs[0].remote_url + repoid = pkgs[0].repoid + repos = filter(lambda r: r.id == repoid, self.repos.listEnabled()) + + if repos: + proxy = repos[0].proxy + if not proxy: + proxy = get_proxy_for(url) + if proxy: + proxies = {str(url.split(':')[0]): str(proxy)} + + return (url, proxies) + + return (None, None) diff --git a/scripts/lib/mic/plugins/backend/zypppkgmgr.py b/scripts/lib/mic/plugins/backend/zypppkgmgr.py new file mode 100755 index 0000000000..c760859832 --- /dev/null +++ b/scripts/lib/mic/plugins/backend/zypppkgmgr.py @@ -0,0 +1,973 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2010, 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil +import urlparse +import rpm + +import zypp +if not hasattr(zypp, 'PoolQuery') or \ + not hasattr(zypp.RepoManager, 'loadSolvFile'): + raise ImportError("python-zypp in host system cannot support PoolQuery or " + "loadSolvFile interface, please update it to enhanced " + "version which can be found in download.tizen.org/tools") + +from mic import msger +from mic.kickstart import ksparser +from mic.utils import misc, rpmmisc, runner, fs_related +from mic.utils.grabber import myurlgrab, TextProgress +from mic.utils.proxy import get_proxy_for +from mic.utils.errors import CreatorError, RepoError, RpmError +from mic.imager.baseimager import BaseImageCreator + +class RepositoryStub: + def __init__(self): + self.name = None + self.baseurl = [] + self.mirrorlist = None + self.proxy = None + self.proxy_username = None + self.proxy_password = None + self.nocache = False + + self.enabled = True + self.autorefresh = True + self.keeppackages = True + self.priority = None + +from mic.pluginbase import BackendPlugin +class Zypp(BackendPlugin): + name = 'zypp' + + def __init__(self, target_arch, instroot, cachedir): + self.cachedir = cachedir + self.instroot = instroot + self.target_arch = target_arch + + self.__pkgs_license = {} + self.__pkgs_content = {} + self.__pkgs_vcsinfo = {} + self.repos = [] + self.to_deselect = [] + self.localpkgs = {} + self.repo_manager = None + self.repo_manager_options = None + self.Z = None + self.ts = None + self.ts_pre = None + self.incpkgs = {} + self.excpkgs = {} + self.pre_pkgs = [] + self.probFilterFlags = [ rpm.RPMPROB_FILTER_OLDPACKAGE, + rpm.RPMPROB_FILTER_REPLACEPKG ] + + self.has_prov_query = True + self.install_debuginfo = False + + def doFileLogSetup(self, uid, logfile): + # don't do the file log for the livecd as it can lead to open fds + # being left and an inability to clean up after ourself + pass + + def closeRpmDB(self): + pass + + def close(self): + if self.ts: + self.ts.closeDB() + self.ts = None + + if self.ts_pre: + self.ts_pre.closeDB() + self.ts = None + + self.closeRpmDB() + + if not os.path.exists("/etc/fedora-release") and \ + not os.path.exists("/etc/meego-release"): + for i in range(3, os.sysconf("SC_OPEN_MAX")): + try: + os.close(i) + except: + pass + + def __del__(self): + self.close() + + def _cleanupRpmdbLocks(self, installroot): + # cleans up temporary files left by bdb so that differing + # versions of rpm don't cause problems + import glob + for f in glob.glob(installroot + "/var/lib/rpm/__db*"): + os.unlink(f) + + def _cleanupZyppJunk(self, installroot): + try: + shutil.rmtree(os.path.join(installroot, '.zypp')) + except: + pass + + def setup(self): + self._cleanupRpmdbLocks(self.instroot) + + def whatObsolete(self, pkg): + query = zypp.PoolQuery() + query.addKind(zypp.ResKind.package) + query.addAttribute(zypp.SolvAttr.obsoletes, pkg) + query.setMatchExact() + for pi in query.queryResults(self.Z.pool()): + return pi + return None + + def _zyppQueryPackage(self, pkg): + query = zypp.PoolQuery() + query.addKind(zypp.ResKind.package) + query.addAttribute(zypp.SolvAttr.name,pkg) + query.setMatchExact() + for pi in query.queryResults(self.Z.pool()): + return pi + return None + + def _splitPkgString(self, pkg): + sp = pkg.rsplit(".",1) + name = sp[0] + arch = None + if len(sp) == 2: + arch = sp[1] + sysarch = zypp.Arch(self.target_arch) + if not zypp.Arch(arch).compatible_with (sysarch): + arch = None + name = ".".join(sp) + return name, arch + + def selectPackage(self, pkg): + """Select a given package or package pattern, can be specified + with name.arch or name* or *name + """ + + if not self.Z: + self.__initialize_zypp() + + def markPoolItem(obs, pi): + if obs == None: + pi.status().setToBeInstalled (zypp.ResStatus.USER) + else: + obs.status().setToBeInstalled (zypp.ResStatus.USER) + + def cmpEVR(p1, p2): + # compare criterion: arch compatibility first, then repo + # priority, and version last + a1 = p1.arch() + a2 = p2.arch() + if str(a1) != str(a2): + if a1.compatible_with(a2): + return -1 + else: + return 1 + # Priority of a repository is an integer value between 0 (the + # highest priority) and 99 (the lowest priority) + pr1 = int(p1.repoInfo().priority()) + pr2 = int(p2.repoInfo().priority()) + if pr1 > pr2: + return -1 + elif pr1 < pr2: + return 1 + + ed1 = p1.edition() + ed2 = p2.edition() + (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()]) + (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()]) + return rpm.labelCompare((e1, v1, r1), (e2, v2, r2)) + + found = False + startx = pkg.startswith("*") + endx = pkg.endswith("*") + ispattern = startx or endx + name, arch = self._splitPkgString(pkg) + + q = zypp.PoolQuery() + q.addKind(zypp.ResKind.package) + + if ispattern: + if startx and not endx: + pattern = '%s$' % (pkg[1:]) + if endx and not startx: + pattern = '^%s' % (pkg[0:-1]) + if endx and startx: + pattern = '%s' % (pkg[1:-1]) + q.setMatchRegex() + q.addAttribute(zypp.SolvAttr.name,pattern) + + elif arch: + q.setMatchExact() + q.addAttribute(zypp.SolvAttr.name,name) + + else: + q.setMatchExact() + q.addAttribute(zypp.SolvAttr.name,pkg) + + for pitem in sorted( + q.queryResults(self.Z.pool()), + cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)), + reverse=True): + item = zypp.asKindPackage(pitem) + if item.name() in self.excpkgs.keys() and \ + self.excpkgs[item.name()] == item.repoInfo().name(): + continue + if item.name() in self.incpkgs.keys() and \ + self.incpkgs[item.name()] != item.repoInfo().name(): + continue + + found = True + obspkg = self.whatObsolete(item.name()) + if arch: + if arch == str(item.arch()): + item.status().setToBeInstalled (zypp.ResStatus.USER) + else: + markPoolItem(obspkg, pitem) + if not ispattern: + break + + # Can't match using package name, then search from packge + # provides infomation + if found == False and not ispattern: + q.addAttribute(zypp.SolvAttr.provides, pkg) + q.addAttribute(zypp.SolvAttr.name,'') + + for pitem in sorted( + q.queryResults(self.Z.pool()), + cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)), + reverse=True): + item = zypp.asKindPackage(pitem) + if item.name() in self.excpkgs.keys() and \ + self.excpkgs[item.name()] == item.repoInfo().name(): + continue + if item.name() in self.incpkgs.keys() and \ + self.incpkgs[item.name()] != item.repoInfo().name(): + continue + + found = True + obspkg = self.whatObsolete(item.name()) + markPoolItem(obspkg, pitem) + break + + if found: + return None + else: + raise CreatorError("Unable to find package: %s" % (pkg,)) + + def inDeselectPackages(self, pitem): + """check if specified pacakges are in the list of inDeselectPackages + """ + item = zypp.asKindPackage(pitem) + name = item.name() + for pkg in self.to_deselect: + startx = pkg.startswith("*") + endx = pkg.endswith("*") + ispattern = startx or endx + pkgname, pkgarch = self._splitPkgString(pkg) + if not ispattern: + if pkgarch: + if name == pkgname and str(item.arch()) == pkgarch: + return True; + else: + if name == pkgname: + return True; + else: + if startx and name.endswith(pkg[1:]): + return True; + if endx and name.startswith(pkg[:-1]): + return True; + + return False; + + def deselectPackage(self, pkg): + """collect packages should not be installed""" + self.to_deselect.append(pkg) + + def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT): + if not self.Z: + self.__initialize_zypp() + found = False + q=zypp.PoolQuery() + q.addKind(zypp.ResKind.pattern) + for pitem in q.queryResults(self.Z.pool()): + item = zypp.asKindPattern(pitem) + summary = "%s" % item.summary() + name = "%s" % item.name() + if name == grp or summary == grp: + found = True + pitem.status().setToBeInstalled (zypp.ResStatus.USER) + break + + if found: + if include == ksparser.GROUP_REQUIRED: + map( + lambda p: self.deselectPackage(p), + grp.default_packages.keys()) + + return None + else: + raise CreatorError("Unable to find pattern: %s" % (grp,)) + + def addRepository(self, name, + url = None, + mirrorlist = None, + proxy = None, + proxy_username = None, + proxy_password = None, + inc = None, + exc = None, + ssl_verify = True, + nocache = False, + cost=None, + priority=None): + # TODO: Handle cost attribute for repos + + if not self.repo_manager: + self.__initialize_repo_manager() + + if not proxy and url: + proxy = get_proxy_for(url) + + repo = RepositoryStub() + repo.name = name + repo.id = name + repo.proxy = proxy + repo.proxy_username = proxy_username + repo.proxy_password = proxy_password + repo.ssl_verify = ssl_verify + repo.nocache = nocache + repo.baseurl.append(url) + if inc: + for pkg in inc: + self.incpkgs[pkg] = name + if exc: + for pkg in exc: + self.excpkgs[pkg] = name + + # check LICENSE files + if not rpmmisc.checkRepositoryEULA(name, repo): + msger.warning('skip repo:%s for failed EULA confirmation' % name) + return None + + if mirrorlist: + repo.mirrorlist = mirrorlist + + # Enable gpg check for verifying corrupt packages + repo.gpgcheck = 1 + if priority is not None: + # priority 0 has issue in RepoInfo.setPriority + repo.priority = priority + 1 + + try: + repo_info = zypp.RepoInfo() + repo_info.setAlias(repo.name) + repo_info.setName(repo.name) + repo_info.setEnabled(repo.enabled) + repo_info.setAutorefresh(repo.autorefresh) + repo_info.setKeepPackages(repo.keeppackages) + baseurl = zypp.Url(repo.baseurl[0]) + if not ssl_verify: + baseurl.setQueryParam("ssl_verify", "no") + if proxy: + scheme, host, path, parm, query, frag = urlparse.urlparse(proxy) + + proxyinfo = host.split(":") + host = proxyinfo[0] + + port = "80" + if len(proxyinfo) > 1: + port = proxyinfo[1] + + if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2: + host = proxy.rsplit(':', 1)[0] + port = proxy.rsplit(':', 1)[1] + + baseurl.setQueryParam ("proxy", host) + baseurl.setQueryParam ("proxyport", port) + + repo.baseurl[0] = baseurl.asCompleteString() + self.repos.append(repo) + + repo_info.addBaseUrl(baseurl) + + if repo.priority is not None: + repo_info.setPriority(repo.priority) + + # this hack is used to change zypp credential file location + # the default one is $HOME/.zypp, which cause conflicts when + # installing some basic packages, and the location doesn't + # have any interface actually, so use a tricky way anyway + homedir = None + if 'HOME' in os.environ: + homedir = os.environ['HOME'] + os.environ['HOME'] = '/' + else: + os.environ['HOME'] = '/' + + self.repo_manager.addRepository(repo_info) + + # save back the $HOME env + if homedir: + os.environ['HOME'] = homedir + else: + del os.environ['HOME'] + + self.__build_repo_cache(name) + + except RuntimeError, e: + raise CreatorError(str(e)) + + msger.verbose('repo: %s was added' % name) + return repo + + def installHasFile(self, file): + return False + + def preInstall(self, pkg): + self.pre_pkgs.append(pkg) + + def runInstall(self, checksize = 0): + os.environ["HOME"] = "/" + os.environ["LD_PRELOAD"] = "" + self.buildTransaction() + + todo = zypp.GetResolvablesToInsDel(self.Z.pool()) + installed_pkgs = todo._toInstall + dlpkgs = [] + for pitem in installed_pkgs: + if not zypp.isKindPattern(pitem) and \ + not self.inDeselectPackages(pitem): + item = zypp.asKindPackage(pitem) + dlpkgs.append(item) + + if not self.install_debuginfo or str(item.arch()) == "noarch": + continue + + dipkg = self._zyppQueryPackage("%s-debuginfo" % item.name()) + if dipkg: + ditem = zypp.asKindPackage(dipkg) + dlpkgs.append(ditem) + else: + msger.warning("No debuginfo rpm found for: %s" \ + % item.name()) + + # record all pkg and the content + localpkgs = self.localpkgs.keys() + for pkg in dlpkgs: + license = '' + if pkg.name() in localpkgs: + hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()]) + pkg_long_name = misc.RPM_FMT % { + 'name': hdr['name'], + 'arch': hdr['arch'], + 'version': hdr['version'], + 'release': hdr['release'] + } + license = hdr['license'] + + else: + pkg_long_name = misc.RPM_FMT % { + 'name': pkg.name(), + 'arch': pkg.arch(), + 'version': pkg.edition().version(), + 'release': pkg.edition().release() + } + + license = pkg.license() + + if license in self.__pkgs_license.keys(): + self.__pkgs_license[license].append(pkg_long_name) + else: + self.__pkgs_license[license] = [pkg_long_name] + + total_count = len(dlpkgs) + cached_count = 0 + download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs)) + localpkgs = self.localpkgs.keys() + + msger.info("Checking packages cached ...") + for po in dlpkgs: + # Check if it is cached locally + if po.name() in localpkgs: + cached_count += 1 + else: + local = self.getLocalPkgPath(po) + name = str(po.repoInfo().name()) + try: + repo = filter(lambda r: r.name == name, self.repos)[0] + except IndexError: + repo = None + nocache = repo.nocache if repo else False + + if os.path.exists(local): + if nocache or self.checkPkg(local) !=0: + os.unlink(local) + else: + download_total_size -= int(po.downloadSize()) + cached_count += 1 + cache_avail_size = misc.get_filesystem_avail(self.cachedir) + if cache_avail_size < download_total_size: + raise CreatorError("No enough space used for downloading.") + + # record the total size of installed pkgs + install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs)) + # check needed size before actually download and install + + # FIXME: for multiple partitions for loop type, check fails + # skip the check temporarily + #if checksize and install_total_size > checksize: + # raise CreatorError("No enough space used for installing, " + # "please resize partition size in ks file") + + download_count = total_count - cached_count + msger.info("Packages: %d Total, %d Cached, %d Missed" \ + % (total_count, cached_count, download_count)) + + try: + if download_count > 0: + msger.info("Downloading packages ...") + self.downloadPkgs(dlpkgs, download_count) + + self.installPkgs(dlpkgs) + + except (RepoError, RpmError): + raise + except Exception, e: + raise CreatorError("Package installation failed: %s" % (e,)) + + def getVcsInfo(self): + if self.__pkgs_vcsinfo: + return + + if not self.ts: + self.__initialize_transaction() + + mi = self.ts.dbMatch() + for hdr in mi: + lname = misc.RPM_FMT % { + 'name': hdr['name'], + 'arch': hdr['arch'], + 'version': hdr['version'], + 'release': hdr['release'] + } + self.__pkgs_vcsinfo[lname] = hdr['VCS'] + + return self.__pkgs_vcsinfo + + def getAllContent(self): + if self.__pkgs_content: + return self.__pkgs_content + + if not self.ts: + self.__initialize_transaction() + + mi = self.ts.dbMatch() + for hdr in mi: + lname = misc.RPM_FMT % { + 'name': hdr['name'], + 'arch': hdr['arch'], + 'version': hdr['version'], + 'release': hdr['release'] + } + self.__pkgs_content[lname] = hdr['FILENAMES'] + + return self.__pkgs_content + + def getPkgsLicense(self): + return self.__pkgs_license + + def getFilelist(self, pkgname): + if not pkgname: + return None + + if not self.ts: + self.__initialize_transaction() + + mi = self.ts.dbMatch('name', pkgname) + for header in mi: + return header['FILENAMES'] + + def __initialize_repo_manager(self): + if self.repo_manager: + return + + # Clean up repo metadata + shutil.rmtree(self.cachedir + "/etc", ignore_errors = True) + shutil.rmtree(self.cachedir + "/solv", ignore_errors = True) + shutil.rmtree(self.cachedir + "/raw", ignore_errors = True) + + zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE + | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED + | zypp.KeyRing.ACCEPT_UNKNOWNKEY + | zypp.KeyRing.TRUST_KEY_TEMPORARILY + ) + + self.repo_manager_options = \ + zypp.RepoManagerOptions(zypp.Pathname(self.instroot)) + + self.repo_manager_options.knownReposPath = \ + zypp.Pathname(self.cachedir + "/etc/zypp/repos.d") + + self.repo_manager_options.repoCachePath = \ + zypp.Pathname(self.cachedir) + + self.repo_manager_options.repoRawCachePath = \ + zypp.Pathname(self.cachedir + "/raw") + + self.repo_manager_options.repoSolvCachePath = \ + zypp.Pathname(self.cachedir + "/solv") + + self.repo_manager_options.repoPackagesCachePath = \ + zypp.Pathname(self.cachedir + "/packages") + + self.repo_manager = zypp.RepoManager(self.repo_manager_options) + + def __build_repo_cache(self, name): + repo = self.repo_manager.getRepositoryInfo(name) + if self.repo_manager.isCached(repo) or not repo.enabled(): + return + + msger.info('Refreshing repository: %s ...' % name) + self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded) + + def __initialize_zypp(self): + if self.Z: + return + + zconfig = zypp.ZConfig_instance() + + # Set system architecture + if self.target_arch: + zconfig.setSystemArchitecture(zypp.Arch(self.target_arch)) + + msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture()) + + # repoPackagesCachePath is corrected by this + self.repo_manager = zypp.RepoManager(self.repo_manager_options) + repos = self.repo_manager.knownRepositories() + for repo in repos: + if not repo.enabled(): + continue + self.repo_manager.loadFromCache(repo) + + self.Z = zypp.ZYppFactory_instance().getZYpp() + self.Z.initializeTarget(zypp.Pathname(self.instroot)) + self.Z.target().load() + + def buildTransaction(self): + if not self.Z.resolver().resolvePool(): + probs = self.Z.resolver().problems() + + for problem in probs: + msger.warning("repo problem: %s, %s" \ + % (problem.description().decode("utf-8"), + problem.details().decode("utf-8"))) + + raise RepoError("found %d resolver problem, abort!" \ + % len(probs)) + + def getLocalPkgPath(self, po): + repoinfo = po.repoInfo() + cacheroot = repoinfo.packagesPath() + location= po.location() + rpmpath = str(location.filename()) + pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath)) + return pkgpath + + def installLocal(self, pkg, po=None, updateonly=False): + if not self.ts: + self.__initialize_transaction() + + solvfile = "%s/.solv" % (self.cachedir) + + rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), + pkg]) + if rc == 0: + f = open(solvfile, "w+") + f.write(out) + f.close() + + warnmsg = self.repo_manager.loadSolvFile(solvfile, + os.path.basename(pkg)) + if warnmsg: + msger.warning(warnmsg) + + os.unlink(solvfile) + else: + msger.warning('Can not get %s solv data.' % pkg) + + hdr = rpmmisc.readRpmHeader(self.ts, pkg) + arch = zypp.Arch(hdr['arch']) + sysarch = zypp.Arch(self.target_arch) + + if arch.compatible_with (sysarch): + pkgname = hdr['name'] + self.localpkgs[pkgname] = pkg + self.selectPackage(pkgname) + msger.info("Marking %s to be installed" % (pkg)) + + else: + msger.warning("Cannot add package %s to transaction. " + "Not a compatible architecture: %s" \ + % (pkg, hdr['arch'])) + + def downloadPkgs(self, package_objects, count): + localpkgs = self.localpkgs.keys() + progress_obj = TextProgress(count) + + for po in package_objects: + if po.name() in localpkgs: + continue + + filename = self.getLocalPkgPath(po) + if os.path.exists(filename): + if self.checkPkg(filename) == 0: + continue + + dirn = os.path.dirname(filename) + if not os.path.exists(dirn): + os.makedirs(dirn) + + url = self.get_url(po) + proxies = self.get_proxies(po) + + try: + filename = myurlgrab(url, filename, proxies, progress_obj) + except CreatorError: + self.close() + raise + + def preinstallPkgs(self): + if not self.ts_pre: + self.__initialize_transaction() + + self.ts_pre.order() + cb = rpmmisc.RPMInstallCallback(self.ts_pre) + cb.headmsg = "Preinstall" + installlogfile = "%s/__catched_stderr.buf" % (self.instroot) + + # start to catch stderr output from librpm + msger.enable_logstderr(installlogfile) + + errors = self.ts_pre.run(cb.callback, '') + # stop catch + msger.disable_logstderr() + self.ts_pre.closeDB() + self.ts_pre = None + + if errors is not None: + if len(errors) == 0: + msger.warning('scriptlet or other non-fatal errors occurred ' + 'during transaction.') + + else: + for e in errors: + msger.warning(e[0]) + raise RepoError('Could not run transaction.') + + def installPkgs(self, package_objects): + if not self.ts: + self.__initialize_transaction() + + # clean rpm lock + self._cleanupRpmdbLocks(self.instroot) + self._cleanupZyppJunk(self.instroot) + # Set filters + probfilter = 0 + for flag in self.probFilterFlags: + probfilter |= flag + self.ts.setProbFilter(probfilter) + self.ts_pre.setProbFilter(probfilter) + + localpkgs = self.localpkgs.keys() + + for po in package_objects: + pkgname = po.name() + if pkgname in localpkgs: + rpmpath = self.localpkgs[pkgname] + else: + rpmpath = self.getLocalPkgPath(po) + + if not os.path.exists(rpmpath): + # Maybe it is a local repo + rpmuri = self.get_url(po) + if rpmuri.startswith("file:/"): + rpmpath = rpmuri[5:] + + if not os.path.exists(rpmpath): + raise RpmError("Error: %s doesn't exist" % rpmpath) + + h = rpmmisc.readRpmHeader(self.ts, rpmpath) + + if pkgname in self.pre_pkgs: + msger.verbose("pre-install package added: %s" % pkgname) + self.ts_pre.addInstall(h, rpmpath, 'u') + + self.ts.addInstall(h, rpmpath, 'u') + + unresolved_dependencies = self.ts.check() + if not unresolved_dependencies: + if self.pre_pkgs: + self.preinstallPkgs() + + self.ts.order() + cb = rpmmisc.RPMInstallCallback(self.ts) + installlogfile = "%s/__catched_stderr.buf" % (self.instroot) + + # start to catch stderr output from librpm + msger.enable_logstderr(installlogfile) + + errors = self.ts.run(cb.callback, '') + # stop catch + msger.disable_logstderr() + self.ts.closeDB() + self.ts = None + + if errors is not None: + if len(errors) == 0: + msger.warning('scriptlet or other non-fatal errors occurred ' + 'during transaction.') + + else: + for e in errors: + msger.warning(e[0]) + raise RepoError('Could not run transaction.') + + else: + for pkg, need, needflags, sense, key in unresolved_dependencies: + package = '-'.join(pkg) + + if needflags == rpm.RPMSENSE_LESS: + deppkg = ' < '.join(need) + elif needflags == rpm.RPMSENSE_EQUAL: + deppkg = ' = '.join(need) + elif needflags == rpm.RPMSENSE_GREATER: + deppkg = ' > '.join(need) + else: + deppkg = '-'.join(need) + + if sense == rpm.RPMDEP_SENSE_REQUIRES: + msger.warning("[%s] Requires [%s], which is not provided" \ + % (package, deppkg)) + + elif sense == rpm.RPMDEP_SENSE_CONFLICTS: + msger.warning("[%s] Conflicts with [%s]" %(package,deppkg)) + + raise RepoError("Unresolved dependencies, transaction failed.") + + def __initialize_transaction(self): + if not self.ts: + self.ts = rpm.TransactionSet(self.instroot) + # Set to not verify DSA signatures. + self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS) + + if not self.ts_pre: + self.ts_pre = rpm.TransactionSet(self.instroot) + # Just unpack the files, don't run scripts + self.ts_pre.setFlags(rpm.RPMTRANS_FLAG_ALLFILES | rpm.RPMTRANS_FLAG_NOSCRIPTS) + # Set to not verify DSA signatures. + self.ts_pre.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS) + + def checkPkg(self, pkg): + ret = 1 + if not os.path.exists(pkg): + return ret + ret = rpmmisc.checkRpmIntegrity('rpm', pkg) + if ret != 0: + msger.warning("package %s is damaged: %s" \ + % (os.path.basename(pkg), pkg)) + + return ret + + def _add_prob_flags(self, *flags): + for flag in flags: + if flag not in self.probFilterFlags: + self.probFilterFlags.append(flag) + + def get_proxies(self, pobj): + if not pobj: + return None + + proxy = None + proxies = None + repoinfo = pobj.repoInfo() + reponame = "%s" % repoinfo.name() + repos = filter(lambda r: r.name == reponame, self.repos) + repourl = str(repoinfo.baseUrls()[0]) + + if repos: + proxy = repos[0].proxy + if not proxy: + proxy = get_proxy_for(repourl) + if proxy: + proxies = {str(repourl.split(':')[0]): str(proxy)} + + return proxies + + def get_url(self, pobj): + if not pobj: + return None + + name = str(pobj.repoInfo().name()) + try: + repo = filter(lambda r: r.name == name, self.repos)[0] + except IndexError: + return None + + baseurl = repo.baseurl[0] + + index = baseurl.find("?") + if index > -1: + baseurl = baseurl[:index] + + location = pobj.location() + location = str(location.filename()) + if location.startswith("./"): + location = location[2:] + + return os.path.join(baseurl, location) + + def package_url(self, pkgname): + + def cmpEVR(p1, p2): + ed1 = p1.edition() + ed2 = p2.edition() + (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()]) + (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()]) + return rpm.labelCompare((e1, v1, r1), (e2, v2, r2)) + + if not self.Z: + self.__initialize_zypp() + + q = zypp.PoolQuery() + q.addKind(zypp.ResKind.package) + q.setMatchExact() + q.addAttribute(zypp.SolvAttr.name, pkgname) + items = sorted(q.queryResults(self.Z.pool()), + cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)), + reverse=True) + + if items: + item = zypp.asKindPackage(items[0]) + url = self.get_url(item) + proxies = self.get_proxies(item) + return (url, proxies) + + return (None, None) diff --git a/scripts/lib/mic/plugins/hook/.py b/scripts/lib/mic/plugins/hook/.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/scripts/lib/mic/plugins/hook/.py diff --git a/scripts/lib/mic/plugins/hook/empty_hook.py b/scripts/lib/mic/plugins/hook/empty_hook.py new file mode 100644 index 0000000000..397585d8c1 --- /dev/null +++ b/scripts/lib/mic/plugins/hook/empty_hook.py @@ -0,0 +1,3 @@ +#!/usr/bin/python + +# TODO: plugin base for hooks diff --git a/scripts/lib/mic/plugins/imager/fs_plugin.py b/scripts/lib/mic/plugins/imager/fs_plugin.py new file mode 100644 index 0000000000..8e758db544 --- /dev/null +++ b/scripts/lib/mic/plugins/imager/fs_plugin.py @@ -0,0 +1,143 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import sys + +from mic import chroot, msger, rt_util +from mic.utils import cmdln, misc, errors, fs_related +from mic.imager import fs +from mic.conf import configmgr +from mic.plugin import pluginmgr + +from mic.pluginbase import ImagerPlugin +class FsPlugin(ImagerPlugin): + name = 'fs' + + @classmethod + @cmdln.option("--include-src", + dest="include_src", + action="store_true", + default=False, + help="Generate a image with source rpms included") + def do_create(self, subcmd, opts, *args): + """${cmd_name}: create fs image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + + if len(args) != 1: + raise errors.Usage("Extra arguments given") + + creatoropts = configmgr.create + ksconf = args[0] + + if creatoropts['runtime'] == 'bootstrap': + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. + if creatoropts['release'] is not None: + creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) + + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = fs.FsImageCreator(creatoropts, pkgmgr) + creator._include_src = opts.include_src + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + self.check_image_exists(creator.destdir, + creator.pack_to, + [creator.name], + creatoropts['release']) + + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + #Download the source packages ###private options + if opts.include_src: + installed_pkgs = creator.get_installed_packages() + msger.info('--------------------------------------------------') + msger.info('Generating the image with source rpms included ...') + if not misc.SrcpkgsDownload(installed_pkgs, creatoropts["repomd"], creator._instroot, creatoropts["cachedir"]): + msger.warning("Source packages can't be downloaded") + + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.package(creatoropts["outdir"]) + if creatoropts['release'] is not None: + creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) + creator.print_outimage_info() + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def do_chroot(self, target, cmd=[]):#chroot.py parse opts&args + try: + if len(cmd) != 0: + cmdline = ' '.join(cmd) + else: + cmdline = "/bin/bash" + envcmd = fs_related.find_binary_inchroot("env", target) + if envcmd: + cmdline = "%s HOME=/root %s" % (envcmd, cmdline) + chroot.chroot(target, None, cmdline) + finally: + chroot.cleanup_after_chroot("dir", None, None, None) + return 1 diff --git a/scripts/lib/mic/plugins/imager/livecd_plugin.py b/scripts/lib/mic/plugins/imager/livecd_plugin.py new file mode 100644 index 0000000000..d24ef59264 --- /dev/null +++ b/scripts/lib/mic/plugins/imager/livecd_plugin.py @@ -0,0 +1,255 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil +import tempfile + +from mic import chroot, msger, rt_util +from mic.utils import misc, fs_related, errors +from mic.conf import configmgr +import mic.imager.livecd as livecd +from mic.plugin import pluginmgr + +from mic.pluginbase import ImagerPlugin +class LiveCDPlugin(ImagerPlugin): + name = 'livecd' + + @classmethod + def do_create(self, subcmd, opts, *args): + """${cmd_name}: create livecd image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + + if len(args) != 1: + raise errors.Usage("Extra arguments given") + + creatoropts = configmgr.create + ksconf = args[0] + + if creatoropts['runtime'] == 'bootstrap': + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + if creatoropts['arch'] and creatoropts['arch'].startswith('arm'): + msger.warning('livecd cannot support arm images, Quit') + return + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. + if creatoropts['release'] is not None: + creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) + + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = livecd.LiveCDImageCreator(creatoropts, pkgmgr) + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + self.check_image_exists(creator.destdir, + creator.pack_to, + [creator.name + ".iso"], + creatoropts['release']) + + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.package(creatoropts["outdir"]) + if creatoropts['release'] is not None: + creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def do_chroot(cls, target, cmd=[]): + os_image = cls.do_unpack(target) + os_image_dir = os.path.dirname(os_image) + + # unpack image to target dir + imgsize = misc.get_file_size(os_image) * 1024L * 1024L + imgtype = misc.get_image_type(os_image) + if imgtype == "btrfsimg": + fstype = "btrfs" + myDiskMount = fs_related.BtrfsDiskMount + elif imgtype in ("ext3fsimg", "ext4fsimg"): + fstype = imgtype[:4] + myDiskMount = fs_related.ExtDiskMount + else: + raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) + + extmnt = misc.mkdtemp() + extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize), + extmnt, + fstype, + 4096, + "%s label" % fstype) + try: + extloop.mount() + + except errors.MountError: + extloop.cleanup() + shutil.rmtree(extmnt, ignore_errors = True) + shutil.rmtree(os_image_dir, ignore_errors = True) + raise + + try: + if len(cmd) != 0: + cmdline = ' '.join(cmd) + else: + cmdline = "/bin/bash" + envcmd = fs_related.find_binary_inchroot("env", extmnt) + if envcmd: + cmdline = "%s HOME=/root %s" % (envcmd, cmdline) + chroot.chroot(extmnt, None, cmdline) + except: + raise errors.CreatorError("Failed to chroot to %s." %target) + finally: + chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt) + + @classmethod + def do_pack(cls, base_on): + import subprocess + + def __mkinitrd(instance): + kernelver = instance._get_kernel_versions().values()[0][0] + args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ] + try: + subprocess.call(args, preexec_fn = instance._chroot) + except OSError, (err, msg): + raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg) + + def __run_post_cleanups(instance): + kernelver = instance._get_kernel_versions().values()[0][0] + args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver] + + try: + subprocess.call(args, preexec_fn = instance._chroot) + except OSError, (err, msg): + raise errors.CreatorError("Failed to run post cleanups: %s" % msg) + + convertoropts = configmgr.convert + convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0] + convertor = livecd.LiveCDImageCreator(convertoropts) + imgtype = misc.get_image_type(base_on) + if imgtype == "btrfsimg": + fstype = "btrfs" + elif imgtype in ("ext3fsimg", "ext4fsimg"): + fstype = imgtype[:4] + else: + raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) + convertor._set_fstype(fstype) + try: + convertor.mount(base_on) + __mkinitrd(convertor) + convertor._create_bootconfig() + __run_post_cleanups(convertor) + convertor.launch_shell(convertoropts['shell']) + convertor.unmount() + convertor.package() + convertor.print_outimage_info() + finally: + shutil.rmtree(os.path.dirname(base_on), ignore_errors = True) + + @classmethod + def do_unpack(cls, srcimg): + img = srcimg + imgmnt = misc.mkdtemp() + imgloop = fs_related.DiskMount(fs_related.LoopbackDisk(img, 0), imgmnt) + try: + imgloop.mount() + except errors.MountError: + imgloop.cleanup() + raise + + # legacy LiveOS filesystem layout support, remove for F9 or F10 + if os.path.exists(imgmnt + "/squashfs.img"): + squashimg = imgmnt + "/squashfs.img" + else: + squashimg = imgmnt + "/LiveOS/squashfs.img" + + tmpoutdir = misc.mkdtemp() + # unsquashfs requires outdir mustn't exist + shutil.rmtree(tmpoutdir, ignore_errors = True) + misc.uncompress_squashfs(squashimg, tmpoutdir) + + try: + # legacy LiveOS filesystem layout support, remove for F9 or F10 + if os.path.exists(tmpoutdir + "/os.img"): + os_image = tmpoutdir + "/os.img" + else: + os_image = tmpoutdir + "/LiveOS/ext3fs.img" + + if not os.path.exists(os_image): + raise errors.CreatorError("'%s' is not a valid live CD ISO : neither " + "LiveOS/ext3fs.img nor os.img exist" %img) + + imgname = os.path.basename(srcimg) + imgname = os.path.splitext(imgname)[0] + ".img" + rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname) + shutil.copyfile(os_image, rtimage) + + finally: + imgloop.cleanup() + shutil.rmtree(tmpoutdir, ignore_errors = True) + shutil.rmtree(imgmnt, ignore_errors = True) + + return rtimage diff --git a/scripts/lib/mic/plugins/imager/liveusb_plugin.py b/scripts/lib/mic/plugins/imager/liveusb_plugin.py new file mode 100644 index 0000000000..7aa8927df9 --- /dev/null +++ b/scripts/lib/mic/plugins/imager/liveusb_plugin.py @@ -0,0 +1,260 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil +import tempfile + +from mic import chroot, msger, rt_util +from mic.utils import misc, fs_related, errors +from mic.utils.partitionedfs import PartitionedMount +from mic.conf import configmgr +from mic.plugin import pluginmgr + +import mic.imager.liveusb as liveusb + +from mic.pluginbase import ImagerPlugin +class LiveUSBPlugin(ImagerPlugin): + name = 'liveusb' + + @classmethod + def do_create(self, subcmd, opts, *args): + """${cmd_name}: create liveusb image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + + if len(args) != 1: + raise errors.Usage("Extra arguments given") + + creatoropts = configmgr.create + ksconf = args[0] + + if creatoropts['runtime'] == "bootstrap": + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + if creatoropts['arch'] and creatoropts['arch'].startswith('arm'): + msger.warning('liveusb cannot support arm images, Quit') + return + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. + if creatoropts['release'] is not None: + creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) + + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = liveusb.LiveUSBImageCreator(creatoropts, pkgmgr) + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + self.check_image_exists(creator.destdir, + creator.pack_to, + [creator.name + ".usbimg"], + creatoropts['release']) + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.package(creatoropts["outdir"]) + if creatoropts['release'] is not None: + creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def do_chroot(cls, target, cmd=[]): + os_image = cls.do_unpack(target) + os_image_dir = os.path.dirname(os_image) + + # unpack image to target dir + imgsize = misc.get_file_size(os_image) * 1024L * 1024L + imgtype = misc.get_image_type(os_image) + if imgtype == "btrfsimg": + fstype = "btrfs" + myDiskMount = fs_related.BtrfsDiskMount + elif imgtype in ("ext3fsimg", "ext4fsimg"): + fstype = imgtype[:4] + myDiskMount = fs_related.ExtDiskMount + else: + raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) + + extmnt = misc.mkdtemp() + extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize), + extmnt, + fstype, + 4096, + "%s label" % fstype) + + try: + extloop.mount() + + except errors.MountError: + extloop.cleanup() + shutil.rmtree(extmnt, ignore_errors = True) + raise + + try: + if len(cmd) != 0: + cmdline = ' '.join(cmd) + else: + cmdline = "/bin/bash" + envcmd = fs_related.find_binary_inchroot("env", extmnt) + if envcmd: + cmdline = "%s HOME=/root %s" % (envcmd, cmdline) + chroot.chroot(extmnt, None, cmdline) + except: + raise errors.CreatorError("Failed to chroot to %s." %target) + finally: + chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt) + + @classmethod + def do_pack(cls, base_on): + import subprocess + + def __mkinitrd(instance): + kernelver = instance._get_kernel_versions().values()[0][0] + args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ] + try: + subprocess.call(args, preexec_fn = instance._chroot) + + except OSError, (err, msg): + raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg) + + def __run_post_cleanups(instance): + kernelver = instance._get_kernel_versions().values()[0][0] + args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver] + + try: + subprocess.call(args, preexec_fn = instance._chroot) + except OSError, (err, msg): + raise errors.CreatorError("Failed to run post cleanups: %s" % msg) + + convertoropts = configmgr.convert + convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0] + convertor = liveusb.LiveUSBImageCreator(convertoropts) + imgtype = misc.get_image_type(base_on) + if imgtype == "btrfsimg": + fstype = "btrfs" + elif imgtype in ("ext3fsimg", "ext4fsimg"): + fstype = imgtype[:4] + else: + raise errors.CreatorError("Unsupported filesystem type: %s" % fstyp) + convertor._set_fstype(fstype) + try: + convertor.mount(base_on) + __mkinitrd(convertor) + convertor._create_bootconfig() + __run_post_cleanups(convertor) + convertor.launch_shell(convertoropts['shell']) + convertor.unmount() + convertor.package() + convertor.print_outimage_info() + finally: + shutil.rmtree(os.path.dirname(base_on), ignore_errors = True) + + @classmethod + def do_unpack(cls, srcimg): + img = srcimg + imgsize = misc.get_file_size(img) * 1024L * 1024L + imgmnt = misc.mkdtemp() + disk = fs_related.SparseLoopbackDisk(img, imgsize) + imgloop = PartitionedMount(imgmnt, skipformat = True) + imgloop.add_disk('/dev/sdb', disk) + imgloop.add_partition(imgsize/1024/1024, "/dev/sdb", "/", "vfat", boot=False) + try: + imgloop.mount() + except errors.MountError: + imgloop.cleanup() + raise + + # legacy LiveOS filesystem layout support, remove for F9 or F10 + if os.path.exists(imgmnt + "/squashfs.img"): + squashimg = imgmnt + "/squashfs.img" + else: + squashimg = imgmnt + "/LiveOS/squashfs.img" + + tmpoutdir = misc.mkdtemp() + # unsquashfs requires outdir mustn't exist + shutil.rmtree(tmpoutdir, ignore_errors = True) + misc.uncompress_squashfs(squashimg, tmpoutdir) + + try: + # legacy LiveOS filesystem layout support, remove for F9 or F10 + if os.path.exists(tmpoutdir + "/os.img"): + os_image = tmpoutdir + "/os.img" + else: + os_image = tmpoutdir + "/LiveOS/ext3fs.img" + + if not os.path.exists(os_image): + raise errors.CreatorError("'%s' is not a valid live CD ISO : neither " + "LiveOS/ext3fs.img nor os.img exist" %img) + imgname = os.path.basename(srcimg) + imgname = os.path.splitext(imgname)[0] + ".img" + rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname) + shutil.copyfile(os_image, rtimage) + + finally: + imgloop.cleanup() + shutil.rmtree(tmpoutdir, ignore_errors = True) + shutil.rmtree(imgmnt, ignore_errors = True) + + return rtimage diff --git a/scripts/lib/mic/plugins/imager/loop_plugin.py b/scripts/lib/mic/plugins/imager/loop_plugin.py new file mode 100644 index 0000000000..8f4b030f6b --- /dev/null +++ b/scripts/lib/mic/plugins/imager/loop_plugin.py @@ -0,0 +1,255 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil +import tempfile + +from mic import chroot, msger, rt_util +from mic.utils import misc, fs_related, errors, cmdln +from mic.conf import configmgr +from mic.plugin import pluginmgr +from mic.imager.loop import LoopImageCreator, load_mountpoints + +from mic.pluginbase import ImagerPlugin +class LoopPlugin(ImagerPlugin): + name = 'loop' + + @classmethod + @cmdln.option("--compress-disk-image", dest="compress_image", + type='choice', choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + # alias to compress-image for compatibility + @cmdln.option("--compress-image", dest="compress_image", + type='choice', choices=("gz", "bz2"), default=None, + help="Compress all loop images with 'gz' or 'bz2'") + @cmdln.option("--shrink", action='store_true', default=False, + help="Whether to shrink loop images to minimal size") + def do_create(self, subcmd, opts, *args): + """${cmd_name}: create loop image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + + if len(args) != 1: + raise errors.Usage("Extra arguments given") + + creatoropts = configmgr.create + ksconf = args[0] + + if creatoropts['runtime'] == "bootstrap": + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # Called After setting the configmgr._ksconf + # as the creatoropts['name'] is reset there. + if creatoropts['release'] is not None: + creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], + creatoropts['release'], + creatoropts['name']) + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = LoopImageCreator(creatoropts, + pkgmgr, + opts.compress_image, + opts.shrink) + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + image_names = [creator.name + ".img"] + image_names.extend(creator.get_image_names()) + self.check_image_exists(creator.destdir, + creator.pack_to, + image_names, + creatoropts['release']) + + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.package(creatoropts["outdir"]) + + if creatoropts['release'] is not None: + creator.release_output(ksconf, + creatoropts['outdir'], + creatoropts['release']) + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def _do_chroot_tar(cls, target, cmd=[]): + mountfp_xml = os.path.splitext(target)[0] + '.xml' + if not os.path.exists(mountfp_xml): + raise errors.CreatorError("No mount point file found for this tar " + "image, please check %s" % mountfp_xml) + + import tarfile + tar = tarfile.open(target, 'r') + tmpdir = misc.mkdtemp() + tar.extractall(path=tmpdir) + tar.close() + + mntdir = misc.mkdtemp() + + loops = [] + for (mp, label, name, size, fstype) in load_mountpoints(mountfp_xml): + if fstype in ("ext2", "ext3", "ext4"): + myDiskMount = fs_related.ExtDiskMount + elif fstype == "btrfs": + myDiskMount = fs_related.BtrfsDiskMount + elif fstype in ("vfat", "msdos"): + myDiskMount = fs_related.VfatDiskMount + else: + msger.error("Cannot support fstype: %s" % fstype) + + name = os.path.join(tmpdir, name) + size = size * 1024L * 1024L + loop = myDiskMount(fs_related.SparseLoopbackDisk(name, size), + os.path.join(mntdir, mp.lstrip('/')), + fstype, size, label) + + try: + msger.verbose("Mount %s to %s" % (mp, mntdir + mp)) + fs_related.makedirs(os.path.join(mntdir, mp.lstrip('/'))) + loop.mount() + + except: + loop.cleanup() + for lp in reversed(loops): + chroot.cleanup_after_chroot("img", lp, None, mntdir) + + shutil.rmtree(tmpdir, ignore_errors=True) + raise + + loops.append(loop) + + try: + if len(cmd) != 0: + cmdline = "/usr/bin/env HOME=/root " + ' '.join(cmd) + else: + cmdline = "/usr/bin/env HOME=/root /bin/bash" + chroot.chroot(mntdir, None, cmdline) + except: + raise errors.CreatorError("Failed to chroot to %s." % target) + finally: + for loop in reversed(loops): + chroot.cleanup_after_chroot("img", loop, None, mntdir) + + shutil.rmtree(tmpdir, ignore_errors=True) + + @classmethod + def do_chroot(cls, target, cmd=[]): + if target.endswith('.tar'): + import tarfile + if tarfile.is_tarfile(target): + LoopPlugin._do_chroot_tar(target, cmd) + return + else: + raise errors.CreatorError("damaged tarball for loop images") + + img = target + imgsize = misc.get_file_size(img) * 1024L * 1024L + imgtype = misc.get_image_type(img) + if imgtype == "btrfsimg": + fstype = "btrfs" + myDiskMount = fs_related.BtrfsDiskMount + elif imgtype in ("ext3fsimg", "ext4fsimg"): + fstype = imgtype[:4] + myDiskMount = fs_related.ExtDiskMount + else: + raise errors.CreatorError("Unsupported filesystem type: %s" \ + % imgtype) + + extmnt = misc.mkdtemp() + extloop = myDiskMount(fs_related.SparseLoopbackDisk(img, imgsize), + extmnt, + fstype, + 4096, + "%s label" % fstype) + try: + extloop.mount() + + except errors.MountError: + extloop.cleanup() + shutil.rmtree(extmnt, ignore_errors=True) + raise + + try: + if len(cmd) != 0: + cmdline = ' '.join(cmd) + else: + cmdline = "/bin/bash" + envcmd = fs_related.find_binary_inchroot("env", extmnt) + if envcmd: + cmdline = "%s HOME=/root %s" % (envcmd, cmdline) + chroot.chroot(extmnt, None, cmdline) + except: + raise errors.CreatorError("Failed to chroot to %s." % img) + finally: + chroot.cleanup_after_chroot("img", extloop, None, extmnt) + + @classmethod + def do_unpack(cls, srcimg): + image = os.path.join(tempfile.mkdtemp(dir="/var/tmp", prefix="tmp"), + "target.img") + msger.info("Copying file system ...") + shutil.copyfile(srcimg, image) + return image diff --git a/scripts/lib/mic/plugins/imager/raw_plugin.py b/scripts/lib/mic/plugins/imager/raw_plugin.py new file mode 100644 index 0000000000..1b9631dfa2 --- /dev/null +++ b/scripts/lib/mic/plugins/imager/raw_plugin.py @@ -0,0 +1,275 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil +import re +import tempfile + +from mic import chroot, msger, rt_util +from mic.utils import misc, fs_related, errors, runner, cmdln +from mic.conf import configmgr +from mic.plugin import pluginmgr +from mic.utils.partitionedfs import PartitionedMount + +import mic.imager.raw as raw + +from mic.pluginbase import ImagerPlugin +class RawPlugin(ImagerPlugin): + name = 'raw' + + @classmethod + @cmdln.option("--compress-disk-image", dest="compress_image", type='choice', + choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + @cmdln.option("--compress-image", dest="compress_image", type='choice', + choices=("gz", "bz2"), default = None, + help="Compress all raw images before package") + @cmdln.option("--generate-bmap", action="store_true", default = None, + help="also generate the block map file") + @cmdln.option("--fstab-entry", dest="fstab_entry", type='choice', + choices=("name", "uuid"), default="uuid", + help="Set fstab entry, 'name' means using device names, " + "'uuid' means using filesystem uuid") + def do_create(self, subcmd, opts, *args): + """${cmd_name}: create raw image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + + if len(args) != 1: + raise errors.Usage("Extra arguments given") + + creatoropts = configmgr.create + ksconf = args[0] + + if creatoropts['runtime'] == "bootstrap": + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. + if creatoropts['release'] is not None: + creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) + + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = raw.RawImageCreator(creatoropts, pkgmgr, opts.compress_image, + opts.generate_bmap, opts.fstab_entry) + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + images = ["%s-%s.raw" % (creator.name, disk_name) + for disk_name in creator.get_disk_names()] + self.check_image_exists(creator.destdir, + creator.pack_to, + images, + creatoropts['release']) + + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.generate_bmap() + creator.package(creatoropts["outdir"]) + if creatoropts['release'] is not None: + creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def do_chroot(cls, target, cmd=[]): + img = target + imgsize = misc.get_file_size(img) * 1024L * 1024L + partedcmd = fs_related.find_binary_path("parted") + disk = fs_related.SparseLoopbackDisk(img, imgsize) + imgmnt = misc.mkdtemp() + imgloop = PartitionedMount(imgmnt, skipformat = True) + imgloop.add_disk('/dev/sdb', disk) + img_fstype = "ext3" + + msger.info("Partition Table:") + partnum = [] + for line in runner.outs([partedcmd, "-s", img, "print"]).splitlines(): + # no use strip to keep line output here + if "Number" in line: + msger.raw(line) + if line.strip() and line.strip()[0].isdigit(): + partnum.append(line.strip()[0]) + msger.raw(line) + + rootpart = None + if len(partnum) > 1: + rootpart = msger.choice("please choose root partition", partnum) + + # Check the partitions from raw disk. + # if choose root part, the mark it as mounted + if rootpart: + root_mounted = True + else: + root_mounted = False + partition_mounts = 0 + for line in runner.outs([partedcmd,"-s",img,"unit","B","print"]).splitlines(): + line = line.strip() + + # Lines that start with number are the partitions, + # because parted can be translated we can't refer to any text lines. + if not line or not line[0].isdigit(): + continue + + # Some vars have extra , as list seperator. + line = line.replace(",","") + + # Example of parted output lines that are handled: + # Number Start End Size Type File system Flags + # 1 512B 3400000511B 3400000000B primary + # 2 3400531968B 3656384511B 255852544B primary linux-swap(v1) + # 3 3656384512B 3720347647B 63963136B primary fat16 boot, lba + + partition_info = re.split("\s+",line) + + size = partition_info[3].split("B")[0] + + if len(partition_info) < 6 or partition_info[5] in ["boot"]: + # No filesystem can be found from partition line. Assuming + # btrfs, because that is the only MeeGo fs that parted does + # not recognize properly. + # TODO: Can we make better assumption? + fstype = "btrfs" + elif partition_info[5] in ["ext2","ext3","ext4","btrfs"]: + fstype = partition_info[5] + elif partition_info[5] in ["fat16","fat32"]: + fstype = "vfat" + elif "swap" in partition_info[5]: + fstype = "swap" + else: + raise errors.CreatorError("Could not recognize partition fs type '%s'." % partition_info[5]) + + if rootpart and rootpart == line[0]: + mountpoint = '/' + elif not root_mounted and fstype in ["ext2","ext3","ext4","btrfs"]: + # TODO: Check that this is actually the valid root partition from /etc/fstab + mountpoint = "/" + root_mounted = True + elif fstype == "swap": + mountpoint = "swap" + else: + # TODO: Assing better mount points for the rest of the partitions. + partition_mounts += 1 + mountpoint = "/media/partition_%d" % partition_mounts + + if "boot" in partition_info: + boot = True + else: + boot = False + + msger.verbose("Size: %s Bytes, fstype: %s, mountpoint: %s, boot: %s" % (size, fstype, mountpoint, boot)) + # TODO: add_partition should take bytes as size parameter. + imgloop.add_partition((int)(size)/1024/1024, "/dev/sdb", mountpoint, fstype = fstype, boot = boot) + + try: + imgloop.mount() + + except errors.MountError: + imgloop.cleanup() + raise + + try: + if len(cmd) != 0: + cmdline = ' '.join(cmd) + else: + cmdline = "/bin/bash" + envcmd = fs_related.find_binary_inchroot("env", imgmnt) + if envcmd: + cmdline = "%s HOME=/root %s" % (envcmd, cmdline) + chroot.chroot(imgmnt, None, cmdline) + except: + raise errors.CreatorError("Failed to chroot to %s." %img) + finally: + chroot.cleanup_after_chroot("img", imgloop, None, imgmnt) + + @classmethod + def do_unpack(cls, srcimg): + srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L + srcmnt = misc.mkdtemp("srcmnt") + disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize) + srcloop = PartitionedMount(srcmnt, skipformat = True) + + srcloop.add_disk('/dev/sdb', disk) + srcloop.add_partition(srcimgsize/1024/1024, "/dev/sdb", "/", "ext3", boot=False) + try: + srcloop.mount() + + except errors.MountError: + srcloop.cleanup() + raise + + image = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), "target.img") + args = ['dd', "if=%s" % srcloop.partitions[0]['device'], "of=%s" % image] + + msger.info("`dd` image ...") + rc = runner.show(args) + srcloop.cleanup() + shutil.rmtree(os.path.dirname(srcmnt), ignore_errors = True) + + if rc != 0: + raise errors.CreatorError("Failed to dd") + else: + return image |