Updated icecc.bbclass

Handles native/cross cross-kernel and target compile jobs
Accepts user supplied package and class exclusion lists

1 files changed, 210 insertions, 93 deletions

diff --git a/classes/icecc.bbclass b/classes/icecc.bbclass
index fbd2814d35..bb5c1b07bf 100644
--- a/classes/icecc.bbclass
+++ b/classes/icecc.bbclass
@@ -5,14 +5,26 @@
 # the directories are added at the head of the PATH list and ICECC_CXX
 # and ICEC_CC are set.
 #
-# For the cross compiler, creates a tar.bz2 of our toolchain and sets
+# For the cross compiler, creates a tar.gz of our toolchain and sets
 # ICECC_VERSION accordingly.
 #
-# This class needs ICECC_PATH to be set already. It must have
-# been exported from the shell running bitbake. Setting it in
-# local.conf is not adequate.
+#The class now handles all 3 different compile 'stages' (i.e native ,cross-kernel and target) creating the
+#necessary enviroment tar.gz file to be used by the remote machines
 #
-# This class objdump, ldconfig, grep, sed installed on the build host.
+#If ICECC_PATH is not set in local.conf then the class will try to locate it using 'which'
+#but nothing is sure ;)
+#
+#If ICECC_ENV_EXEC is set in local.conf should point to the icecc-create-env script provided by the user
+#or the default one provided by icecc-create-env.bb  will be used
+#(NOTE that this is a modified version of the script need it and *not the one that comes with icecc*
+#
+#User can specify if specific packages or packages belonging to class should not use icecc to distribute
+#compile jobs to remote machines, but handled localy, by defining ICECC_USER_CLASS_BL and ICECC_PACKAGE_BL
+#with the appropriate values in local.conf
+#########################################################################################
+#Error checking is kept to minimum so double check any parameters you pass to the class
+###########################################################################################
+
 
 def icc_determine_gcc_version(gcc):
     """
@@ -23,7 +35,7 @@ def icc_determine_gcc_version(gcc):
     import os
     return os.popen("%s --version" % gcc ).readline().split()[2]
 
-def create_env(bb,d):
+def create_cross_env(bb,d):
     """
     Create a tar.bz2 of the current toolchain
     """
@@ -40,20 +52,61 @@ def create_env(bb,d):
     distro  = bb.data.expand('${DISTRO}', d)
     target_sys = bb.data.expand('${TARGET_SYS}',  d)
     target_prefix = bb.data.expand('${TARGET_PREFIX}',  d)
-    float   = bb.data.getVar('${TARGET_FPU}', d) or "hard"
+    float   = bb.data.getVar('TARGET_FPU', d) or "hard"
     name    = socket.gethostname()
+  
 
     # Stupid check to determine if we have built a libc and a cross
     # compiler.
     try:
-        os.stat(os.path.join(ice_dir, target_sys, 'lib', 'ld-linux.so.2'))
+        os.stat(os.path.join(ice_dir, target_sys, 'lib', 'libc.so'))
         os.stat(os.path.join(ice_dir, target_sys, 'bin', 'g++'))
     except: # no cross compiler built yet
         return ""
 
     VERSION = icc_determine_gcc_version( os.path.join(ice_dir,target_sys,"bin","g++") )
-    cross_name = prefix + distro + target_sys + float +VERSION+ name
-    tar_file = os.path.join(ice_dir, 'ice', cross_name + '.tar.bz2')
+    cross_name = prefix + distro + "-" + target_sys + "-" + float + "-" + VERSION + "-" + name
+    tar_file = os.path.join(ice_dir, 'ice', cross_name + '.tar.gz')
+
+    try:
+        os.stat(tar_file)
+	# tar file already exists
+        return tar_file
+    except: 
+        try:
+            os.makedirs(os.path.join(ice_dir,'ice'))
+        except:
+            # directory already exists, continue
+            pass
+
+
+    #check if user has specified a specific icecc-create-env script
+    #if not use the OE provided one
+    cr_env_script = bb.data.getVar('ICECC_ENV_EXEC',  d) or  bb.data.expand('${STAGING_DIR}', d)+"/ice/icecc-create-env"
+    #call the modified create-env script
+    result=os.popen("%s %s %s %s %s %s" %(cr_env_script,
+           "--silent",
+           os.path.join(ice_dir,target_sys,'bin','gcc'),
+           os.path.join(ice_dir,target_sys,'bin','g++'),
+           os.path.join(ice_dir,target_sys,'bin','as'),
+           os.path.join(ice_dir,"ice",cross_name) ) )
+    return tar_file
+
+
+def create_native_env(bb,d):
+
+    import tarfile, socket, time, os
+    ice_dir = bb.data.expand('${CROSS_DIR}', d)
+    prefix  = bb.data.expand('${HOST_PREFIX}' , d)
+    distro  = bb.data.expand('${DISTRO}', d)
+    target_sys = bb.data.expand('${TARGET_SYS}',  d)
+    target_prefix = bb.data.expand('${TARGET_PREFIX}',  d)
+    float   = bb.data.getVar('TARGET_FPU', d) or "hard"
+    name    = socket.gethostname()
+  
+    
+    archive_name = "local-host-env" + "-" + name
+    tar_file = os.path.join(ice_dir, 'ice', archive_name + '.tar.gz')
 
     try:
         os.stat(tar_file)
@@ -61,79 +114,88 @@ def create_env(bb,d):
         return tar_file
     except: 
         try:
+            #os.makedirs(os.path.join(ice_dir))
             os.makedirs(os.path.join(ice_dir,'ice'))
         except:
             # directory already exists, continue
             pass
 
-    # FIXME find out the version of the compiler
-    # Consider using -print-prog-name={cc1,cc1plus}
-    # and            -print-file-name=specs
-
-    # We will use the GCC to tell us which tools to use
-    #  What we need is:
-    #        -gcc
-    #        -g++
-    #        -as
-    #        -cc1
-    #        -cc1plus
-    #  and we add them to /usr/bin
-
-    tar = tarfile.open(tar_file, 'w:bz2')
-
-    # Now add the required files
-    tar.add(os.path.join(ice_dir,target_sys,'bin','gcc'),
-            os.path.join("usr","bin","gcc") )
-    tar.add(os.path.join(ice_dir,target_sys,'bin','g++'),
-            os.path.join("usr","bin","g++") )
-    tar.add(os.path.join(ice_dir,target_sys,'bin','as'),
-            os.path.join("usr","bin","as") )
-
-    cc = bb.data.getVar('CC', d, True)
-
-    # use bitbake's PATH so that the cross-compiler is actually found on the PATH
-    oldpath = os.environ['PATH']
-    os.environ['PATH'] = bb.data.getVar('PATH', d, True)
-
-    # FIXME falsely assuming there is only a single NEEDED per file
-    # FIXME falsely assuming the lib path is /lib
-
-    # which libc does the compiler need? (for example: libc.so.6)
-    libc = os.popen("objdump -x `which %s` | sed -n 's/.*NEEDED *//p'" % cc).read()[:-1]
-    # what is the absolute path of libc? (for example: /lib/libc.so.6)
-    # FIXME assuming only one entry is returned, which easily breaks
-    libc = os.popen("ldconfig -p | grep -e %s$ | sed 's:[^/]*/:/:'" % libc).read()[:-1]
-
-    # which loader does the compiler need?
-    ldlinux = os.popen("objdump -x %s | sed -n 's/.*NEEDED *//p'" % libc).read()[:-1]
-    ldlinux = os.popen("ldconfig -p | grep -e %s$ | sed 's:[^/]*/:/:'" % ldlinux).read()[:-1]
-
-    tar.add(libc)
-    tar.add(ldlinux)
+
+    #check if user has specified a specific icecc-create-env script
+    #if not use the OE provided one
+    cr_env_script = bb.data.getVar('ICECC_ENV_EXEC',  d) or  bb.data.expand('${STAGING_DIR}', d)+"/ice/icecc-create-env"
+    result=os.popen("%s %s %s %s %s %s" %(cr_env_script,
+           "--silent",
+           os.popen("%s gcc" % "which").read()[:-1],
+           os.popen("%s g++" % "which").read()[:-1],
+           os.popen("%s as" % "which").read()[:-1],
+           os.path.join(ice_dir,"ice",archive_name) ) )
+    return tar_file
+
+
+
+def create_cross_kernel_env(bb,d):
+
+    import tarfile, socket, time, os
+    ice_dir = bb.data.expand('${CROSS_DIR}', d)
+    prefix  = bb.data.expand('${HOST_PREFIX}' , d)
+    distro  = bb.data.expand('${DISTRO}', d)
+    target_sys = bb.data.expand('${TARGET_SYS}',  d)
+    target_prefix = bb.data.expand('${TARGET_PREFIX}',  d)
+    float   = bb.data.getVar('TARGET_FPU', d) or "hard"
+    name    = socket.gethostname()
+    kernel_cc = bb.data.expand('${KERNEL_CC}', d)
+    kernel_cc = kernel_cc[:-1]
   
-    # Now let us find cc1 and cc1plus
-    cc1 = os.popen("%s -print-prog-name=cc1" % cc).read()[:-1]
-    cc1plus = os.popen("%s -print-prog-name=cc1plus" % cc).read()[:-1]
-    spec = os.popen("%s -print-file-name=specs" % cc).read()[:-1]
 
-    os.environ['PATH'] = oldpath
+    # Stupid check to determine if we have built a libc and a cross
+    # compiler.
+    try:
+       os.stat(os.path.join(ice_dir, 'bin', kernel_cc))
+    except: # no cross compiler built yet
+        return ""
 
-    # CC1 and CC1PLUS should be there...
-    #tar.add(cc1, os.path.join('usr', 'bin', 'cc1'))
-    #tar.add(cc1plus, os.path.join('usr', 'bin', 'cc1plus'))
+    VERSION = icc_determine_gcc_version( os.path.join(ice_dir,"bin",kernel_cc) )
+    cross_name = prefix + distro + "-" + target_sys + "-" + float + "-" + VERSION + "-" + name
+    tar_file = os.path.join(ice_dir, 'ice', cross_name + '.tar.gz')
 
-    # I think they should remain absolute paths (as gcc expects them there)
-    tar.add(cc1)
-    tar.add(cc1plus)
+    try:
+        os.stat(tar_file)
+	# tar file already exists
+        return tar_file
+    except: 
+        try:
+            os.makedirs(os.path.join(ice_dir,'ice'))
+        except:
+            # directory already exists, continue
+            pass
 
-    # spec - if it exists
-    if os.path.exists(spec):
-        tar.add(spec)
 
-    tar.close()
+    #check if user has specified a specific icecc-create-env script
+    #if not use the OE provided one
+    cr_env_script = bb.data.getVar('ICECC_ENV_EXEC',  d) or  bb.data.expand('${STAGING_DIR}', d)+"/ice/icecc-create-env"
+    result=os.popen("%s %s %s %s %s %s" %(cr_env_script,
+           "--silent",
+           os.path.join(ice_dir,'bin',kernel_cc),
+           os.path.join(ice_dir,target_sys,'bin','g++'),
+           os.path.join(ice_dir,target_sys,'bin','as'),
+           os.path.join(ice_dir,"ice",cross_name) ) )
     return tar_file
 
 
+def create_env(bb,d):
+
+        #return create_cross_kernel_env(bb,d) 
+        if bb.data.inherits_class("native", d):
+          return create_native_env(bb,d)
+        elif bb.data.inherits_class("kernel", d):
+          return create_cross_kernel_env(bb,d)
+        elif bb.data.inherits_class("cross", d):
+          return create_native_env(bb,d)
+        else:  
+          return create_cross_env(bb,d)
+        
+       
 def create_path(compilers, type, bb, d):
     """
     Create Symlinks for the icecc in the staging directory
@@ -141,8 +203,11 @@ def create_path(compilers, type, bb, d):
     import os
 
     staging = os.path.join(bb.data.expand('${STAGING_DIR}', d), "ice", type)
-    icecc   = bb.data.getVar('ICECC_PATH', d)
 
+    #check if the icecc path is set by the user
+    icecc   = bb.data.getVar('ICECC_PATH', d) or os.popen("%s icecc" % "which").read()[:-1]
+
+    
     # Create the dir if necessary
     try:
         os.stat(staging)
@@ -158,42 +223,79 @@ def create_path(compilers, type, bb, d):
 
     return staging + ":"
 
+
+
+
+
 def use_icc_version(bb,d):
-    # Constin native native
-    prefix = bb.data.expand('${HOST_PREFIX}', d)
-    if len(prefix) == 0:
-        return "no"
 
-    blacklist = [ "cross", "native" ]
+      icecc_ver = "yes"
+      system_class_blacklist = [ "none" ] 
+      
+      user_class_blacklist =  bb.data.getVar('ICECC_USER_CLASS_BL', d) or "none"
+      system_class_blacklist = user_class_blacklist.split()
+
+      for black in system_class_blacklist:
+           print("Class value is %s " %black)
+           if bb.data.inherits_class(black, d):
+              icecc_ver = "no"
+ 
+      return icecc_ver
 
-    for black in blacklist:
-        if bb.data.inherits_class(black, d):
-            return "no"
 
-    return "yes"
 
 def icc_path(bb,d,compile):
-    native = bb.data.expand('${PN}', d)
-    blacklist = [ "ulibc", "glibc", "ncurses" ]
-    for black in blacklist:
-        if black in native:
-            return ""
+    package_tmp = bb.data.expand('${PN}', d)
+
+    #"system" package blacklist contains a list of packages that can not distribute compile tasks
+    #for one reason or the other
+    system_package_blacklist = [ "ulibc", "glibc", "qemu" ]
+
+    #user defined exclusion list
+    user_package_blacklist = bb.data.getVar('ICECC_USER_PACKAGE_BL', d) or "none"   
+    system_package_blacklist = user_package_blacklist.split()
+
+    for black in system_package_blacklist:
+      if black in package_tmp:
+         return ""
 
-    blacklist = [ "cross", "native" ]
-    for black in blacklist:
-        if bb.data.inherits_class(black, d):
-            compile = False
 
     prefix = bb.data.expand('${HOST_PREFIX}', d)
-    if compile and len(prefix) != 0:
-        return create_path( [prefix+"gcc", prefix+"g++"], "cross", bb, d)
+
+            
+    if compile and bb.data.inherits_class("cross", d):
+       return create_path( ["gcc", "g++"], "native", bb, d)
+
+    elif compile and bb.data.inherits_class("native", d):
+         return create_path( ["gcc", "g++"], "native", bb, d)
+
+    elif compile and bb.data.inherits_class("kernel", d):
+         #kernel_cc = bb.data.expand('${KERNEL_CC}', d) 
+          return create_path( [get_cross_kernel_ver(bb,d), "foo"], "cross-kernel", bb, d)
+
     elif not compile or len(prefix) == 0:
-        return create_path( ["gcc", "g++"], "native", bb, d)
+           return create_path( ["gcc", "g++"], "native", bb, d)
+
+    else:
+           return create_path( [prefix+"gcc", prefix+"g++"], "cross", bb, d)      
+
+
+
 
 def icc_version(bb,d):
     return create_env(bb,d)
 
-#
+def check_for_kernel(bb,d):     
+     if  bb.data.inherits_class("kernel", d):
+       return "yes"
+
+       return "no"
+
+
+def get_cross_kernel_ver(bb,d):
+
+       return  bb.data.expand('${KERNEL_CC}', d).strip() or "gcc"
+ 
 # set the icecream environment variables
 do_configure_prepend() {
     export PATH=${@icc_path(bb,d,False)}$PATH
@@ -202,12 +304,27 @@ do_configure_prepend() {
 }
 
 do_compile_prepend() {
+
     export PATH=${@icc_path(bb,d,True)}$PATH
+
+ #check if we are building a kernel and select gcc-cross-kernel
+ if [ "${@check_for_kernel(bb,d)}" = "yes" ]; then
+    export ICECC_CC="${@get_cross_kernel_ver(bb,d)}"
+    export ICECC_CXX="${HOST_PREFIX}g++"
+ else
     export ICECC_CC="${HOST_PREFIX}gcc"
     export ICECC_CXX="${HOST_PREFIX}g++"
+ fi
 
     if [ "${@use_icc_version(bb,d)}" = "yes" ]; then
-        print ICECC_VERSION="${@icc_version(bb,d)}"
         export ICECC_VERSION="${@icc_version(bb,d)}"
+    else
+        export ICECC_VERSION="NONE"
     fi
 }
+
+
+
+
+
+DEPENDS =+ "icecc-create-env"