recipetool: create: support extracting name and version from build scripts

Some build systems (notably autotools) support declaring the name and
version of the program being built; since we need those for the recipe
we can attempt to extract them. It's a little fuzzy as they are often
omitted or may not be appropriately formatted for our purposes, but it
does work on a reasonable number of software packages to be useful.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 237 insertions, 45 deletions

diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 931ef3b33f..0aff59e229 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -17,7 +17,7 @@
 
 import re
 import logging
-from recipetool.create import RecipeHandler, read_pkgconfig_provides
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
 
 logger = logging.getLogger('recipetool')
 
@@ -27,13 +27,17 @@ def tinfoil_init(instance):
     global tinfoil
     tinfoil = instance
 
+
 class CmakeRecipeHandler(RecipeHandler):
-    def process(self, srctree, classes, lines_before, lines_after, handled):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:
             return False
 
         if RecipeHandler.checkfiles(srctree, ['CMakeLists.txt']):
             classes.append('cmake')
+            values = CmakeRecipeHandler.extract_cmake_deps(lines_before, srctree, extravalues)
+            for var, value in values.iteritems():
+                lines_before.append('%s = "%s"' % (var, value))
             lines_after.append('# Specify any options you want to pass to cmake using EXTRA_OECMAKE:')
             lines_after.append('EXTRA_OECMAKE = ""')
             lines_after.append('')
@@ -41,8 +45,26 @@ class CmakeRecipeHandler(RecipeHandler):
             return True
         return False
 
+    @staticmethod
+    def extract_cmake_deps(outlines, srctree, extravalues, cmakelistsfile=None):
+        values = {}
+
+        if cmakelistsfile:
+            srcfiles = [cmakelistsfile]
+        else:
+            srcfiles = RecipeHandler.checkfiles(srctree, ['CMakeLists.txt'])
+
+        proj_re = re.compile('project\(([^)]*)\)', re.IGNORECASE)
+        with open(srcfiles[0], 'r') as f:
+            for line in f:
+                res = proj_re.match(line.strip())
+                if res:
+                    extravalues['PN'] = res.group(1).split()[0]
+
+        return values
+
 class SconsRecipeHandler(RecipeHandler):
-    def process(self, srctree, classes, lines_before, lines_after, handled):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:
             return False
 
@@ -56,7 +78,7 @@ class SconsRecipeHandler(RecipeHandler):
         return False
 
 class QmakeRecipeHandler(RecipeHandler):
-    def process(self, srctree, classes, lines_before, lines_after, handled):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:
             return False
 
@@ -67,14 +89,14 @@ class QmakeRecipeHandler(RecipeHandler):
         return False
 
 class AutotoolsRecipeHandler(RecipeHandler):
-    def process(self, srctree, classes, lines_before, lines_after, handled):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:
             return False
 
         autoconf = False
         if RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in']):
             autoconf = True
-            values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree)
+            values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree, extravalues)
             classes.extend(values.pop('inherit', '').split())
             for var, value in values.iteritems():
                 lines_before.append('%s = "%s"' % (var, value))
@@ -88,6 +110,22 @@ class AutotoolsRecipeHandler(RecipeHandler):
                             autoconf = True
                             break
 
+        if autoconf and not ('PV' in extravalues and 'PN' in extravalues):
+            # Last resort
+            conffile = RecipeHandler.checkfiles(srctree, ['configure'])
+            if conffile:
+                with open(conffile[0], 'r') as f:
+                    for line in f:
+                        line = line.strip()
+                        if line.startswith('VERSION=') or line.startswith('PACKAGE_VERSION='):
+                            pv = line.split('=')[1].strip('"\'')
+                            if pv and not 'PV' in extravalues and validate_pv(pv):
+                                extravalues['PV'] = pv
+                        elif line.startswith('PACKAGE_NAME=') or line.startswith('PACKAGE='):
+                            pn = line.split('=')[1].strip('"\'')
+                            if pn and not 'PN' in extravalues:
+                                extravalues['PN'] = pn
+
         if autoconf:
             lines_before.append('# NOTE: if this software is not capable of being built in a separate build directory')
             lines_before.append('# from the source, you should replace autotools with autotools-brokensep in the')
@@ -102,7 +140,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
         return False
 
     @staticmethod
-    def extract_autotools_deps(outlines, srctree, acfile=None):
+    def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None):
         import shlex
         import oe.package
 
@@ -122,6 +160,9 @@ class AutotoolsRecipeHandler(RecipeHandler):
         lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*')
         progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)\]?[),].*')
         dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
+        ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*')
+        am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*')
+        define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)')
 
         # Build up lib library->package mapping
         shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data)
@@ -157,55 +198,157 @@ class AutotoolsRecipeHandler(RecipeHandler):
                 else:
                     raise
 
+        defines = {}
+        def subst_defines(value):
+            newvalue = value
+            for define, defval in defines.iteritems():
+                newvalue = newvalue.replace(define, defval)
+            if newvalue != value:
+                return subst_defines(newvalue)
+            return value
+
+        def process_value(value):
+            value = value.replace('[', '').replace(']', '')
+            if value.startswith('m4_esyscmd(') or value.startswith('m4_esyscmd_s('):
+                cmd = subst_defines(value[value.index('(')+1:-1])
+                try:
+                    if '|' in cmd:
+                        cmd = 'set -o pipefail; ' + cmd
+                    stdout, _ = bb.process.run(cmd, cwd=srctree, shell=True)
+                    ret = stdout.rstrip()
+                except bb.process.ExecutionError as e:
+                    ret = ''
+            elif value.startswith('m4_'):
+                return None
+            ret = subst_defines(value)
+            if ret:
+                ret = ret.strip('"\'')
+            return ret
+
         # Since a configure.ac file is essentially a program, this is only ever going to be
         # a hack unfortunately; but it ought to be enough of an approximation
         if acfile:
             srcfiles = [acfile]
         else:
-            srcfiles = RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in'])
+            srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in'])
+
         pcdeps = []
         deps = []
         unmapped = []
         unmappedlibs = []
-        with open(srcfiles[0], 'r') as f:
-            for line in f:
-                if 'PKG_CHECK_MODULES' in line:
-                    res = pkg_re.search(line)
-                    if res:
-                        res = dep_re.findall(res.group(1))
-                        if res:
-                            pcdeps.extend([x[0] for x in res])
-                    inherits.append('pkgconfig')
-                if line.lstrip().startswith('AM_GNU_GETTEXT'):
-                    inherits.append('gettext')
-                elif 'AC_CHECK_PROG' in line or 'AC_PATH_PROG' in line:
-                    res = progs_re.search(line)
+
+        def process_macro(keyword, value):
+            if keyword == 'PKG_CHECK_MODULES':
+                res = pkg_re.search(value)
+                if res:
+                    res = dep_re.findall(res.group(1))
                     if res:
-                        for prog in shlex.split(res.group(1)):
-                            prog = prog.split()[0]
-                            progclass = progclassmap.get(prog, None)
-                            if progclass:
-                                inherits.append(progclass)
+                        pcdeps.extend([x[0] for x in res])
+                inherits.append('pkgconfig')
+            elif keyword == 'AM_GNU_GETTEXT':
+                inherits.append('gettext')
+            elif keyword == 'AC_CHECK_PROG' or keyword == 'AC_PATH_PROG':
+                res = progs_re.search(value)
+                if res:
+                    for prog in shlex.split(res.group(1)):
+                        prog = prog.split()[0]
+                        progclass = progclassmap.get(prog, None)
+                        if progclass:
+                            inherits.append(progclass)
+                        else:
+                            progdep = progmap.get(prog, None)
+                            if progdep:
+                                deps.append(progdep)
                             else:
-                                progdep = progmap.get(prog, None)
-                                if progdep:
-                                    deps.append(progdep)
-                                else:
-                                    if not prog.startswith('$'):
-                                        unmapped.append(prog)
-                elif 'AC_CHECK_LIB' in line:
-                    res = lib_re.search(line)
+                                if not prog.startswith('$'):
+                                    unmapped.append(prog)
+            elif keyword == 'AC_CHECK_LIB':
+                res = lib_re.search(value)
+                if res:
+                    lib = res.group(1)
+                    libdep = recipelibmap.get(lib, None)
+                    if libdep:
+                        deps.append(libdep)
+                    else:
+                        if libdep is None:
+                            if not lib.startswith('$'):
+                                unmappedlibs.append(lib)
+            elif keyword == 'AC_PATH_X':
+                deps.append('libx11')
+            elif keyword == 'AC_INIT':
+                if extravalues is not None:
+                    res = ac_init_re.match(value)
                     if res:
-                        lib = res.group(1)
-                        libdep = recipelibmap.get(lib, None)
-                        if libdep:
-                            deps.append(libdep)
-                        else:
-                            if libdep is None:
-                                if not lib.startswith('$'):
-                                    unmappedlibs.append(lib)
-                elif 'AC_PATH_X' in line:
-                    deps.append('libx11')
+                        extravalues['PN'] = process_value(res.group(1))
+                        pv = process_value(res.group(2))
+                        if validate_pv(pv):
+                            extravalues['PV'] = pv
+            elif keyword == 'AM_INIT_AUTOMAKE':
+                if extravalues is not None:
+                    if 'PN' not in extravalues:
+                        res = am_init_re.match(value)
+                        if res:
+                            if res.group(1) != 'AC_PACKAGE_NAME':
+                                extravalues['PN'] = process_value(res.group(1))
+                            pv = process_value(res.group(2))
+                            if validate_pv(pv):
+                                extravalues['PV'] = pv
+            elif keyword == 'define(':
+                res = define_re.match(value)
+                if res:
+                    key = res.group(2).strip('[]')
+                    value = process_value(res.group(3))
+                    if value is not None:
+                        defines[key] = value
+
+        keywords = ['PKG_CHECK_MODULES',
+                    'AM_GNU_GETTEXT',
+                    'AC_CHECK_PROG',
+                    'AC_PATH_PROG',
+                    'AC_CHECK_LIB',
+                    'AC_PATH_X',
+                    'AC_INIT',
+                    'AM_INIT_AUTOMAKE',
+                    'define(',
+                    ]
+        for srcfile in srcfiles:
+            nesting = 0
+            in_keyword = ''
+            partial = ''
+            with open(srcfile, 'r') as f:
+                for line in f:
+                    if in_keyword:
+                        partial += ' ' + line.strip()
+                        if partial.endswith('\\'):
+                            partial = partial[:-1]
+                        nesting = nesting + line.count('(') - line.count(')')
+                        if nesting == 0:
+                            process_macro(in_keyword, partial)
+                            partial = ''
+                            in_keyword = ''
+                    else:
+                        for keyword in keywords:
+                            if keyword in line:
+                                nesting = line.count('(') - line.count(')')
+                                if nesting > 0:
+                                    partial = line.strip()
+                                    if partial.endswith('\\'):
+                                        partial = partial[:-1]
+                                    in_keyword = keyword
+                                else:
+                                    process_macro(keyword, line.strip())
+                                break
+
+            if in_keyword:
+                process_macro(in_keyword, partial)
+
+        if extravalues:
+            for k,v in extravalues.items():
+                if v:
+                    if v.startswith('$') or v.startswith('@') or v.startswith('%'):
+                        del extravalues[k]
+                    else:
+                        extravalues[k] = v.strip('"\'').rstrip('()')
 
         if unmapped:
             outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(unmapped))
@@ -240,7 +383,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
 
 
 class MakefileRecipeHandler(RecipeHandler):
-    def process(self, srctree, classes, lines_before, lines_after, handled):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:
             return False
 
@@ -307,6 +450,53 @@ class MakefileRecipeHandler(RecipeHandler):
             self.genfunction(lines_after, 'do_install', ['# Specify install commands here'])
 
 
+class VersionFileRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        if 'PV' not in extravalues:
+            # Look for a VERSION or version file containing a single line consisting
+            # only of a version number
+            filelist = RecipeHandler.checkfiles(srctree, ['VERSION', 'version'])
+            version = None
+            for fileitem in filelist:
+                linecount = 0
+                with open(fileitem, 'r') as f:
+                    for line in f:
+                        line = line.rstrip().strip('"\'')
+                        linecount += 1
+                        if line:
+                            if linecount > 1:
+                                version = None
+                                break
+                            else:
+                                if validate_pv(line):
+                                    version = line
+                if version:
+                    extravalues['PV'] = version
+                    break
+
+
+class SpecFileRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        if 'PV' in extravalues and 'PN' in extravalues:
+            return
+        filelist = RecipeHandler.checkfiles(srctree, ['*.spec'], recursive=True)
+        pn = None
+        pv = None
+        for fileitem in filelist:
+            linecount = 0
+            with open(fileitem, 'r') as f:
+                for line in f:
+                    if line.startswith('Name:') and not pn:
+                        pn = line.split(':')[1].strip()
+                    if line.startswith('Version:') and not pv:
+                        pv = line.split(':')[1].strip()
+            if pv or pn:
+                if pv and not 'PV' in extravalues and validate_pv(pv):
+                    extravalues['PV'] = pv
+                if pn and not 'PN' in extravalues:
+                    extravalues['PN'] = pn
+                break
+
 def register_recipe_handlers(handlers):
     # These are in a specific order so that the right one is detected first
     handlers.append(CmakeRecipeHandler())
@@ -314,3 +504,5 @@ def register_recipe_handlers(handlers):
     handlers.append(SconsRecipeHandler())
     handlers.append(QmakeRecipeHandler())
     handlers.append(MakefileRecipeHandler())
+    handlers.append((VersionFileRecipeHandler(), -1))
+    handlers.append((SpecFileRecipeHandler(), -1))