utility-tasks.bbclass: Move distro related tasks to distrodata.bbclass

Most of the d.keys() used in file parsing are variables in
distro_tracking_fields.inc, which are not used in normal build.
Therefore remove the inclusion of distro_tracking_fields.inc from
poky.conf. Besides, move distro related tasks to distrodata.bbclass,
which includes that tracking field file.
By this change, the file parsing time could save about 25%.

Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>

1 files changed, 0 insertions, 442 deletions

diff --git a/meta/classes/utility-tasks.bbclass b/meta/classes/utility-tasks.bbclass
index 205a206baf..3ab37fa6b9 100644
--- a/meta/classes/utility-tasks.bbclass
+++ b/meta/classes/utility-tasks.bbclass
@@ -47,396 +47,6 @@ python do_rebuild() {
 #	bb.build.exec_func('do_clean', d)
 #}
 
-addtask checkpkg
-do_checkpkg[nostamp] = "1"
-python do_checkpkg() {
-	import sys
-	import re
-	import tempfile
-
-	"""
-	sanity check to ensure same name and type. Match as many patterns as possible
-	such as:
-		gnome-common-2.20.0.tar.gz (most common format)
-		gtk+-2.90.1.tar.gz
-		xf86-intput-synaptics-12.6.9.tar.gz
-		dri2proto-2.3.tar.gz
-		blktool_4.orig.tar.gz
-		libid3tag-0.15.1b.tar.gz
-		unzip552.tar.gz
-		icu4c-3_6-src.tgz
-		genext2fs_1.3.orig.tar.gz
-		gst-fluendo-mp3
-	"""
-	prefix1 = "[a-zA-Z][a-zA-Z0-9]*([\-_][a-zA-Z]\w+)*[\-_]"	# match most patterns which uses "-" as separator to version digits
-	prefix2 = "[a-zA-Z]+"			# a loose pattern such as for unzip552.tar.gz
-	prefix = "(%s|%s)" % (prefix1, prefix2)
-	suffix = "(tar\.gz|tgz|tar\.bz2|zip)"
-	suffixtuple = ("tar.gz", "tgz", "zip", "tar.bz2")
-
-	sinterstr = "(?P<name>%s?)(?P<ver>.*)" % prefix
-	sdirstr = "(?P<name>%s)(?P<ver>.*)\.(?P<type>%s$)" % (prefix, suffix)
-
-	def parse_inter(s):
-		m = re.search(sinterstr, s)
-		if not m:
-			return None
-		else:
-			return (m.group('name'), m.group('ver'), "")
-
-	def parse_dir(s):
-		m = re.search(sdirstr, s)
-		if not m:
-			return None
-		else:
-			return (m.group('name'), m.group('ver'), m.group('type'))
-
-	"""
-	Check whether 'new' is newer than 'old' version. We use existing vercmp() for the
-	purpose. PE is cleared in comparison as it's not for build, and PV is cleared too
-	for simplicity as it's somehow difficult to get from various upstream format
-	"""
-	def __vercmp(old, new):
-		(on, ov, ot) = old
-		(en, ev, et) = new
-		if on != en or (et and et not in suffixtuple):
-			return 0
-
-		ov = re.search("\d+[^a-zA-Z]+", ov).group()
-		ev = re.search("\d+[^a-zA-Z]+", ev).group()
-		return bb.utils.vercmp(("0", ov, ""), ("0", ev, ""))
-
-	"""
-	wrapper for fetch upstream directory info
-		'url'	- upstream link customized by regular expression
-		'd'	- database
-		'tmpf'	- tmpfile for fetcher output
-	We don't want to exit whole build due to one recipe error. So handle all exceptions 
-	gracefully w/o leaking to outer. 
-	"""
-	def internal_fetch_wget(url, d, tmpf):
-		status = "ErrFetchUnknown"
-		try: 
-			"""
-			Clear internal url cache as it's a temporary check. Not doing so will have 
-			bitbake check url multiple times when looping through a single url
-			"""
-			fn = bb.data.getVar('FILE', d, 1)
-			bb.fetch.urldata_cache[fn] = {}
-			bb.fetch.init([url], d)
-		except bb.fetch.NoMethodError:
-			status = "ErrFetchNoMethod"
-		except:
-			status = "ErrInitUrlUnknown"
-		else:
-			"""
-			To avoid impacting bitbake build engine, this trick is required for reusing bitbake
-			interfaces. bb.fetch.go() is not appliable as it checks downloaded content in ${DL_DIR}
-			while we don't want to pollute that place. So bb.fetch.checkstatus() is borrowed here
-			which is designed for check purpose but we override check command for our own purpose
-			"""
-			ld = bb.data.createCopy(d)
-			bb.data.setVar('CHECKCOMMAND_wget', "/usr/bin/env wget -t 1 --passive-ftp -O %s '${URI}'" \
-					% tmpf.name, d)
-			bb.data.update_data(ld)
-
-			try:
-				bb.fetch.checkstatus(ld)
-			except bb.fetch.MissingParameterError:
-				status = "ErrMissParam"
-			except bb.fetch.FetchError:
-				status = "ErrFetch"
-			except bb.fetch.MD5SumError:
-				status = "ErrMD5Sum"
-			except:
-				status = "ErrFetchUnknown"
-			else:
-				status = "SUCC" 
-		return status
-
-	"""
-	Check on middle version directory such as "2.4/" in "http://xxx/2.4/pkg-2.4.1.tar.gz", 
-		'url'	- upstream link customized by regular expression
-		'd'	- database
-		'curver' - current version
-	Return new version if success, or else error in "Errxxxx" style
-	"""
-	def check_new_dir(url, curver, d):
-		pn = bb.data.getVar('PN', d, 1)
-		f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-1-" % pn)
-		status = internal_fetch_wget(url, d, f)
-		fhtml = f.read()
-
-		if status == "SUCC" and len(fhtml):
-			newver = parse_inter(curver)
-
-			"""
-			match "*4.1/">*4.1/ where '*' matches chars
-			N.B. add package name, only match for digits
-			"""
-			m = re.search("^%s" % prefix, curver)
-			if m:
-				s = "%s[^\d\"]*?(\d+[\.\-_])+\d+/?" % m.group()
-			else:
-				s = "(\d+[\.\-_])+\d+/?"
-				
-			searchstr = "[hH][rR][eE][fF]=\"%s\">" % s
-			reg = re.compile(searchstr)
-
-			valid = 0
-			for line in fhtml.split("\n"):
-				if line.find(curver) >= 0:
-					valid = 1
-
-				m = reg.search(line)
-				if m:
-					ver = m.group().split("\"")[1]
-					ver = ver.strip("/")
-					ver = parse_inter(ver)
-					if ver and __vercmp(newver, ver) < 0:
-						newver = ver
-
-			"""Expect a match for curver in directory list, or else it indicates unknown format"""
-			if not valid:
-				status = "ErrParseInterDir"
-			else:
-				"""rejoin the path name"""
-				status = newver[0] + newver[1]
-		elif not len(fhtml):
-			status = "ErrHostNoDir"
-
-		f.close()
-		if status != "ErrHostNoDir" and re.match("Err", status):
-			logpath = bb.data.getVar('LOG_DIR', d, 1)
-			os.system("cp %s %s/" % (f.name, logpath))
-		os.unlink(f.name)
-		return status
-
-	"""
-	Check on the last directory to search '2.4.1' in "http://xxx/2.4/pkg-2.4.1.tar.gz", 
-		'url'	- upstream link customized by regular expression
-		'd'	- database
-		'curname' - current package name
-	Return new version if success, or else error in "Errxxxx" style
-	"""
-	def check_new_version(url, curname, d):
-		"""possible to have no version in pkg name, such as spectrum-fw"""
-		if not re.search("\d+", curname):
-			return pcurver
-		pn = bb.data.getVar('PN', d, 1)
-		f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-2-" % pn)
-		status = internal_fetch_wget(url, d, f)
-		fhtml = f.read()
-
-		if status == "SUCC" and len(fhtml):
-			newver = parse_dir(curname)
-
-			"""match "{PN}-5.21.1.tar.gz">{PN}-5.21.1.tar.gz """
-			pn1 = re.search("^%s" % prefix, curname).group()
-			s = "[^\"]*%s[^\d\"]*?(\d+[\.\-_])+[^\"]*" % pn1
-			searchstr = "[hH][rR][eE][fF]=\"%s\">" % s
-			reg = re.compile(searchstr)
-	
-			valid = 0
-			for line in fhtml.split("\n"):
-				m = reg.search(line)
-				if m:
-					valid = 1
-					ver = m.group().split("\"")[1].split("/")[-1]
-					ver = parse_dir(ver)
-					if ver and __vercmp(newver, ver) < 0:
-						newver = ver
-	
-			"""Expect a match for curver in directory list, or else it indicates unknown format"""
-			if not valid:
-				status = "ErrParseDir"
-			else:
-				"""newver still contains a full package name string"""
-				status = re.search("(\d+[.\-_])*\d+", newver[1]).group()
-		elif not len(fhtml):
-			status = "ErrHostNoDir"
-
-		f.close()
-		"""if host hasn't directory information, no need to save tmp file"""
-		if status != "ErrHostNoDir" and re.match("Err", status):
-			logpath = bb.data.getVar('LOG_DIR', d, 1)
-			os.system("cp %s %s/" % (f.name, logpath))
-		os.unlink(f.name)
-		return status
-
-	"""first check whether a uri is provided"""
-	src_uri = bb.data.getVar('SRC_URI', d, 1)
-	if not src_uri:
-		return
-
-	"""initialize log files."""
-	logpath = bb.data.getVar('LOG_DIR', d, 1)
-	bb.utils.mkdirhier(logpath)
-	logfile = os.path.join(logpath, "poky_pkg_info.log.%s" % bb.data.getVar('DATETIME', d, 1))
-	if not os.path.exists(logfile):
-		slogfile = os.path.join(logpath, "poky_pkg_info.log")
-		if os.path.exists(slogfile):
-			os.remove(slogfile)
-		os.system("touch %s" % logfile)
-		os.symlink(logfile, slogfile)
-
-	"""generate package information from .bb file"""
-	pname = bb.data.getVar('PN', d, 1)
-	pdesc = bb.data.getVar('DESCRIPTION', d, 1)
-	pgrp = bb.data.getVar('SECTION', d, 1)
-
-	found = 0
-	for uri in src_uri.split():
-		m = re.compile('(?P<type>[^:]*)').match(uri)
-		if not m:
-			raise MalformedUrl(uri)
-		elif m.group('type') in ('http', 'https', 'ftp', 'cvs', 'svn', 'git'):
-			found = 1
-			pproto = m.group('type')
-			break
-	if not found:
-		pproto = "file"
-	pupver = "N/A"
-	pstatus = "ErrUnknown"
-
-	(type, host, path, user, pswd, parm) = bb.decodeurl(uri)
-	if type in ['http', 'https', 'ftp']:
-		pcurver = bb.data.getVar('PV', d, 1)
-	else:
-		pcurver = bb.data.getVar("SRCREV", d, 1)
-
-	if type in ['http', 'https', 'ftp']:
-		newver = pcurver
-		altpath = path
-		dirver = "-"
-		curname = "-"
-	
-		"""
-		match version number amid the path, such as "5.7" in:
-			http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz	
-		N.B. how about sth. like "../5.7/5.8/..."? Not find such example so far :-P
-		"""
-		m = re.search(r"[^/]*(\d+\.)+\d+([\-_]r\d+)*/", path)
-		if m:
-			altpath = path.split(m.group())[0]
-			dirver = m.group().strip("/")
-	
-			"""use new path and remove param. for wget only param is md5sum"""
-			alturi = bb.encodeurl([type, host, altpath, user, pswd, {}])
-	
-			newver = check_new_dir(alturi, dirver, d)
-			altpath = path
-			if not re.match("Err", newver) and dirver != newver:
-				altpath = altpath.replace(dirver, newver, 1)
-				
-		"""Now try to acquire all remote files in current directory"""
-		if not re.match("Err", newver):
-			curname = altpath.split("/")[-1]
-	
-			"""get remote name by skipping pacakge name"""
-			m = re.search(r"/.*/", altpath)
-			if not m:
-				altpath = "/"
-			else:
-				altpath = m.group()
-	
-			alturi = bb.encodeurl([type, host, altpath, user, pswd, {}])
-			newver = check_new_version(alturi, curname, d)
-			if not re.match("Err", newver):
-				pupver = newver
-				if pupver != pcurver:
-					pstatus = "UPDATE"
-				else:
-					pstatus = "MATCH"
-	
-		if re.match("Err", newver):
-			pstatus = newver + ":" + altpath + ":" + dirver + ":" + curname
-	elif type == 'git':
-		if user:
-			gituser = user + '@'
-		else:
-			gituser = ""
-
-		if 'protocol' in parm:
-			gitproto = parm['protocol']
-		else:
-			gitproto = "rsync"
-
-		gitcmd = "git ls-remote %s://%s%s%s HEAD 2>&1" % (gitproto, gituser, host, path)
-		print gitcmd
-		ver = os.popen(gitcmd).read()
-		if ver and re.search("HEAD", ver):
-			pupver = ver.split("\t")[0]
-			if pcurver == pupver:
-				pstatus = "MATCH"
-			else:
-				pstatus = "UPDATE"
-		else:
-			pstatus = "ErrGitAccess"
-	elif type == 'svn':
-		options = []
-		if user:
-			options.append("--username %s" % user)
-		if pswd:
-			options.append("--password %s" % pswd)
-		svnproto = 'svn'
-		if 'proto' in parm:
-			svnproto = parm['proto']
-		if 'rev' in parm:
-			pcurver = parm['rev']
-
-		svncmd = "svn info %s %s://%s%s/%s/ 2>&1" % (" ".join(options), svnproto, host, path, parm["module"])
-		print svncmd
-		svninfo = os.popen(svncmd).read()
-		for line in svninfo.split("\n"):
-			if re.search("^Last Changed Rev:", line):
-				pupver = line.split(" ")[-1]
-				if pcurver == pupver:
-					pstatus = "MATCH"
-				else:
-					pstatus = "UPDATE"
-
-		if re.match("Err", pstatus):
-			pstatus = "ErrSvnAccess"
-	elif type == 'cvs':
-		pupver = "HEAD"
-		pstatus = "UPDATE"
-	elif type == 'file':
-		"""local file is always up-to-date"""
-		pupver = pcurver
-		pstatus = "MATCH"
-	else:
-		pstatus = "ErrUnsupportedProto"
-
-	if re.match("Err", pstatus):
-		pstatus += ":%s%s" % (host, path)
-
-	"""Read from manual distro tracking fields as alternative"""
-	pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, 1)
-	if not pmver:
-		pmver = "N/A"
-		pmstatus = "ErrNoRecipeData"
-	else:
-		if pmver == pcurver:
-			pmstatus = "MATCH"
-		else:
-			pmstatus = "UPDATE"
-	
-	lf = bb.utils.lockfile(logfile + ".lock")
-	f = open(logfile, "a")
-	f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \
-		  (pname, pgrp, pproto, pcurver, pmver, pupver, pmstatus, pstatus, pdesc))
-	f.close()
-	bb.utils.unlockfile(lf)
-}
-
-addtask checkpkgall after do_checkpkg
-do_checkpkgall[recrdeptask] = "do_checkpkg"
-do_checkpkgall[nostamp] = "1"
-do_checkpkgall() {
-	:
-}
-
 addtask checkuri
 do_checkuri[nostamp] = "1"
 python do_checkuri() {
@@ -487,55 +97,3 @@ do_buildall[recrdeptask] = "do_build"
 do_buildall() {
 	:
 }
-
-#addhandler check_eventhandler
-python check_eventhandler() {
-    from bb.event import Handled, NotHandled
-    # if bb.event.getName(e) == "TaskStarted":
-
-    if bb.event.getName(e) == "BuildStarted":
-        import oe.distro_check as dc
-        tmpdir = bb.data.getVar('TMPDIR', e.data, 1)
-        distro_check_dir = os.path.join(tmpdir, "distro_check")
-        datetime = bb.data.getVar('DATETIME', e.data, 1)
-        """initialize log files."""
-        logpath = bb.data.getVar('LOG_DIR', e.data, 1)
-        bb.utils.mkdirhier(logpath)
-        logfile = os.path.join(logpath, "distrocheck.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
-        if not os.path.exists(logfile):
-                slogfile = os.path.join(logpath, "distrocheck.csv")
-                if os.path.exists(slogfile):
-                        os.remove(slogfile)
-                os.system("touch %s" % logfile)
-                os.symlink(logfile, slogfile)
-                bb.data.setVar('LOG_FILE', logfile, e.data)
-
-    return NotHandled
-}
-
-addtask distro_check
-do_distro_check[nostamp] = "1"
-python do_distro_check() {
-    """checks if the package is present in other public Linux distros"""
-    import oe.distro_check as dc
-    localdata = bb.data.createCopy(d)
-    bb.data.update_data(localdata)
-    tmpdir = bb.data.getVar('TMPDIR', d, 1)
-    distro_check_dir = os.path.join(tmpdir, "distro_check")
-    datetime = bb.data.getVar('DATETIME', localdata, 1)
-    dc.update_distro_data(distro_check_dir, datetime)
-
-    # do the comparison
-    result = dc.compare_in_distro_packages_list(distro_check_dir, d)
-
-    # save the results
-    dc.save_distro_check_result(result, datetime, d)
-}
-
-addtask distro_checkall after do_distro_check
-do_distro_checkall[recrdeptask] = "do_distro_check"
-do_distro_checkall[nostamp] = "1"
-do_distro_checkall() {
-	:
-}
-