lib/bb/parse/parser_c: Handle includes, requires, cope with our syntax

    -be able to parse variables like PREFERRED_PROVIDER_virtual/kernel
    -handle includes and requires, add 's' to %(fn)
    -be less verbose

4 files changed, 103 insertions, 97 deletions

diff --git a/lib/bb/parse/parse_c/BBHandler.py b/lib/bb/parse/parse_c/BBHandler.py
index fd3be268e..e15efa982 100644
--- a/lib/bb/parse/parse_c/BBHandler.py
+++ b/lib/bb/parse/parse_c/BBHandler.py
@@ -70,12 +70,12 @@ def init(fn, data):
 def handle(fn, d, include):
     from bb import data, parse
 
-    print ""
-    print "fn: %s" % fn
-    print "data: %s" % d
-    print dir(d)
-    print d.getVar.__doc__
-    print "include: %s" % include
+    #print ""
+    #print "fn: %s" % fn
+    #print "data: %s" % d
+    #print dir(d)
+    #print d.getVar.__doc__
+    print "include: %s" % fn
 
     # check if we include or are the beginning
     if include:
@@ -93,7 +93,7 @@ def handle(fn, d, include):
 
     # check if the file exists
     if not os.path.exists(abs_fn):
-        raise IOError("file '%(fn)' not found" % locals() )
+        raise IOError("file '%(fn)s' not found" % locals() )
 
     # now we know the file is around mark it as dep
     if include:
diff --git a/lib/bb/parse/parse_c/bitbakec.pyx b/lib/bb/parse/parse_c/bitbakec.pyx
index 6bdad36e0..b26ebceac 100644
--- a/lib/bb/parse/parse_c/bitbakec.pyx
+++ b/lib/bb/parse/parse_c/bitbakec.pyx
@@ -22,45 +22,45 @@ cdef extern from "lexerc.h":
     cdef extern void parse(FILE*, object)
 
 def parsefile(object file, object data):
-    print "parsefile: 1", file, data
+    #print "parsefile: 1", file, data
 
     # Open the file
     cdef FILE* f
 
     f = fopen(file, "r")
-    print "parsefile: 2 opening file"
+    #print "parsefile: 2 opening file"
     if (f == NULL):
         raise IOError("No such file %s." % file)
 
-    print "parsefile: 3 parse"
+    #print "parsefile: 3 parse"
     parse(f, data)
 
     # Close the file
-    print "parsefile: 4 closing"
+    #print "parsefile: 4 closing"
     fclose(f)
 
 
 cdef public void e_assign(lex_t* container, char* key, char* what):
-    print "e_assign", key, what
+    #print "e_assign", key, what
     d = <object>container.data
     d.setVar(key, what)
 
 cdef public void e_export(lex_t* c, char* what):
-    print "e_export", what
+    #print "e_export", what
     #exp:
     # bb.data.setVarFlag(key, "export", 1, data)
     d = <object>c.data
     d.setVarFlag(what, "export", 1)
 
 cdef public void e_immediate(lex_t* c, char* key, char* what):
-    print "e_immediate", key, what
+    #print "e_immediate", key, what
     #colon:
     # val = bb.data.expand(groupd["value"], data)
     d = <object>c.data
     d.setVar(key, d.expand(what,None))
 
 cdef public void e_cond(lex_t* c, char* key, char* what):
-    print "e_cond", key, what
+    #print "e_cond", key, what
     #ques:
     # val = bb.data.getVar(key, data)
     # if val == None:    
@@ -69,35 +69,35 @@ cdef public void e_cond(lex_t* c, char* key, char* what):
     d.setVar(key, (d.getVar(key,0) or what))
 
 cdef public void e_prepend(lex_t* c, char* key, char* what):
-    print "e_prepend", key, what
+    #print "e_prepend", key, what
     #prepend:
     # val = "%s %s" % (groupd["value"], (bb.data.getVar(key, data) or ""))
     d = <object>c.data
     d.setVar(key, what + " " + (d.getVar(key,0) or ""))
 
 cdef public void e_append(lex_t* c, char* key, char* what):
-    print "e_append", key, what
+    #print "e_append", key, what
     #append:
     # val = "%s %s" % ((bb.data.getVar(key, data) or ""), groupd["value"])
     d = <object>c.data
     d.setVar(key, (d.getVar(key,0) or "") + " " + what)
 
 cdef public void e_precat(lex_t* c, char* key, char* what):
-    print "e_precat", key, what
+    #print "e_precat", key, what
     #predot:
     # val = "%s%s" % (groupd["value"], (bb.data.getVar(key, data) or ""))
     d = <object>c.data
     d.setVar(key, what + (d.getVar(key,0) or ""))
 
 cdef public void e_postcat(lex_t* c, char* key, char* what):
-    print "e_postcat", key, what
+    #print "e_postcat", key, what
     #postdot:
     # val = "%s%s" % ((bb.data.getVar(key, data) or ""), groupd["value"])
     d = <object>c.data
     d.setVar(key, (d.getVar(key,0) or "") + what)
 
 cdef public void e_addtask(lex_t* c, char* name, char* before, char* after):
-    print "e_addtask", name
+    #print "e_addtask", name
     # func = m.group("func")
     # before = m.group("before")
     # after = m.group("after")
@@ -120,69 +120,75 @@ cdef public void e_addtask(lex_t* c, char* name, char* before, char* after):
     d.setVarFlag(do, "task", 1)
 
     if before != NULL and strlen(before) > 0:
-        print "Before", before
+        #print "Before", before
         d.setVarFlag(do, "postdeps", ("%s" % before).split())
     if after  != NULL and strlen(after) > 0:
-        print "After", after
+        #print "After", after
         d.setVarFlag(do, "deps", ("%s" % after).split())
 
 
 cdef public void e_addhandler(lex_t* c, char* h):
-    print "e_addhandler", h
+    #print "e_addhandler", h
     # data.setVarFlag(h, "handler", 1, d)
     d = <object>c.data
     d.setVarFlag(h, "handler", 1)
 
 cdef public void e_export_func(lex_t* c, char* function):
-    print "e_export_func", function
+    #print "e_export_func", function
     pass
 
 cdef public void e_inherit(lex_t* c, char* file):
-    print "e_inherit", file
+    #print "e_inherit", file
     pass
 
 cdef public void e_include(lex_t* c, char* file):
-    print "e_include", file
     from bb.parse import handle
     d = <object>c.data
 
     try:
-        handle(d.expand(file,None), d, True)
+        handle(d.expand(file,d), d, True)
     except IOError:
-        print "Could not include required file %s" % file
+        print "Could not include file", file
 
 
 cdef public void e_require(lex_t* c, char* file):
-    print "e_require", file
+    #print "e_require", file
+    from bb.parse import handle
     d = <object>c.data
-    d.expand(file)
 
     try:
-        parsefile(file, d)
+        handle(d.expand(file,d), d, True)
     except IOError:
-        raise CParseError("Could not include required file %s" % file)
+        print "ParseError", file
+        from bb.parse import ParseError
+        raise ParseError("Could not include required file %s" % file)
 
 cdef public void e_proc(lex_t* c, char* key, char* what):
-    print "e_proc", key, what
+    #print "e_proc", key, what
     pass
 
 cdef public void e_proc_python(lex_t* c, char* key, char* what):
-    print "e_proc_python"
+    #print "e_proc_python"
     if key != NULL:
-        print "Key", key
+        pass
+        #print "Key", key
     if what != NULL:
-        print "What", what
+        pass
+        #print "What", what
     pass
 
 cdef public void e_proc_fakeroot(lex_t* c, char* key, char* what):
-    print "e_fakeroot", key, what
+    #print "e_fakeroot", key, what
     pass
 
 cdef public void e_def(lex_t* c, char* a, char* b, char* d):
-    print "e_def", a, b, d
+    #print "e_def", a, b, d
     pass
 
 cdef public void e_parse_error(lex_t* c):
     print "e_parse_error", "line:", lineError, "parse:", errorParse
-    raise CParseError("There was an parse error, sorry unable to give more information at the current time.")
+
+
+    from bb.parse import ParseError
+    raise ParseError("There was an parse error, sorry unable to give more information at the current time.")
 
diff --git a/lib/bb/parse/parse_c/bitbakescanner.cc b/lib/bb/parse/parse_c/bitbakescanner.cc
index 43dad12d3..1aeca06ab 100644
--- a/lib/bb/parse/parse_c/bitbakescanner.cc
+++ b/lib/bb/parse/parse_c/bitbakescanner.cc
@@ -493,7 +493,7 @@ static yyconst flex_int32_t yy_ec[256] =
 static yyconst flex_int32_t yy_meta[59] =
     {   0,
         1,    1,    2,    3,    1,    1,    4,    1,    1,    1,
-        5,    6,    5,    5,    7,    8,    1,    7,    1,    9,
+        5,    6,    5,    5,    5,    7,    1,    8,    1,    9,
         9,    9,    9,    9,    9,    9,    9,    9,    9,    9,
         9,    9,   10,    1,   11,    9,    9,    9,    9,    9,
         9,    9,    9,    9,    9,    9,    9,    9,    9,    9,
@@ -565,18 +565,18 @@ static yyconst flex_int16_t yy_base[847] =
      2077, 2072, 2066, 2069, 2056, 2067, 1398, 1343, 1408, 1404,
       643, 1409, 2071, 2066, 2060, 2063, 2050, 2061, 2065, 2060,
      2054, 2057, 2044, 2055, 1420, 1445, 1413, 1447, 1453, 1454,
-     2059, 2054, 2047, 2050, 2035, 2043, 1455, 1459, 1460, 1461,
+     2059, 2053, 2047, 2049, 2032, 2043, 1455, 1459, 1460, 1461,
      1462, 1463, 1471, 1436, 1430, 1192, 1433, 1479, 1482, 1492,
 
-     1506, 1519, 1520, 1528, 2047, 2040, 2031,    0, 2034, 2019,
-     2027, 1486, 1496, 1505, 1506, 1510, 1516, 1524, 2044, 2018,
-        0,    0,    0,    0, 1281, 1517, 2043, 2042, 2039, 2035,
-     2023, 1994, 2309, 2309, 2309, 2309, 2005, 1981,    0,    0,
+     1506, 1519, 1520, 1528, 2046, 2037, 2031,    0, 2033, 2016,
+     2027, 1486, 1496, 1505, 1506, 1510, 1516, 1524, 2043, 2015,
+        0,    0,    0,    0, 1281, 1517, 2043, 2041, 2036, 2034,
+     2024, 1995, 2309, 2309, 2309, 2309, 2005, 1981,    0,    0,
         0,    0, 1538, 1528, 1530, 1534, 1537, 1540, 1981, 1957,
         0,    0,    0,    0, 1557, 1558, 1559, 1560, 1561, 1563,
-     1568, 1547, 1988, 1959, 1954, 1948, 1580, 1581, 1582, 1590,
-     1592, 1594, 1923, 1863,    0,    0,    0,    0, 1598, 1599,
-     1600, 1874, 1858, 1350, 1584, 1803, 1792, 1801, 1790, 1603,
+     1568, 1547, 1988, 1959, 1955, 1948, 1580, 1581, 1582, 1590,
+     1592, 1594, 1924, 1863,    0,    0,    0,    0, 1598, 1599,
+     1600, 1875, 1859, 1350, 1584, 1803, 1792, 1801, 1790, 1603,
      1601, 1799, 1788, 1604, 1602, 1610, 1609, 1643, 1644, 1797,
 
      1786, 1611, 1630, 1800, 1773, 1010, 1606, 1798, 1771, 1795,
@@ -593,8 +593,8 @@ static yyconst flex_int16_t yy_base[847] =
      1768,    0,  742, 2309,    0, 1764,    0, 1778,  678, 1801,
         0, 2309, 1835, 1847, 1859, 1871, 1883,  550, 1892, 1898,
      1907, 1919, 1931, 1939, 1945, 1950, 1956, 1965, 1977, 1989,
-     2001, 2013, 2025, 2033, 2039, 2042,  306,  304,  301, 2049,
-      213, 2057,  136, 2065, 2073, 2081
+     2001, 2013, 2025, 2033, 2039, 2043,  306,  304,  301, 2050,
+      213, 2058,  136, 2066, 2074, 2082
     } ;
 
 static yyconst flex_int16_t yy_def[847] =
@@ -903,14 +903,14 @@ static yyconst flex_int16_t yy_nxt[2368] =
       112,  112,  112,  112,  112,  112,  112,  112,  112,  112,
       112,  128,  128,  128,  128,  128,  128,  128,  128,  128,
       128,  128,  128,  155,  155,  155,  155,  155,  155,  155,
-      155,  155,  155,  155,  155,  167,  167,  167,  705,  167,
+      155,  155,  155,  155,  155,  167,  167,  167,  167,  705,
 
-      167,  167,  177,  177,  704,  177,  177,  183,  701,  183,
+      167,  167,  177,  177,  177,  704,  177,  183,  701,  183,
       183,  183,  183,  183,  183,  183,  183,  183,  183,  187,
       187,  187,  187,  187,  187,  187,  187,  187,  187,  187,
       187,  201,  201,  201,  201,  201,  201,  201,  201,  201,
-      201,  201,  201,  209,  209,  700,  209,  209,  217,  217,
-      238,  217,  217,  217,  223,  223,  238,  223,  223,  231,
+      201,  201,  201,  209,  209,  209,  700,  209,  217,  217,
+      238,  217,  217,  217,  223,  223,  223,  238,  223,  231,
       231,  238,  231,  231,  231,  237,  237,  237,  237,  237,
       237,  237,  237,  237,  237,  237,  237,  239,  239,  239,
       239,  239,  239,  239,  239,  239,  239,  239,  239,  256,
@@ -919,13 +919,13 @@ static yyconst flex_int16_t yy_nxt[2368] =
       256,  261,  693,  692,  261,  261,  261,  261,  261,  261,
       261,  261,  261,  264,  264,  264,  264,  264,  264,  264,
       264,  264,  264,  264,  264,  267,  689,  688,  267,  267,
-      267,  267,  267,  267,  267,  267,  267,  284,  284,  687,
-      284,  284,  292,  292,  292,  686,  292,  292,  292,  296,
-      296,  184,  296,  418,  418,  184,  418,  418,  184,  184,
-      418,  433,  433,  683,  433,  433,  682,  678,  433,  465,
-      465,  677,  465,  465,  676,  675,  465,  500,  500,  674,
-      500,  500,  673,  654,  500,  514,  514,  653,  514,  514,
-      652,  651,  514,  650,  649,  642,  641,  640,  639,  638,
+      267,  267,  267,  267,  267,  267,  267,  284,  284,  284,
+      687,  284,  292,  292,  292,  292,  686,  292,  292,  296,
+      184,  296,  184,  296,  418,  418,  418,  184,  418,  184,
+      683,  418,  433,  433,  433,  682,  433,  678,  677,  433,
+      465,  465,  465,  676,  465,  675,  674,  465,  500,  500,
+      500,  673,  500,  654,  653,  500,  514,  514,  514,  652,
+      514,  651,  650,  514,  649,  642,  641,  640,  639,  638,
 
       637,  636,  635,  634,  633,  632,  631,  624,  623,  622,
       621,  620,  619,  611,  610,  609,  608,  607,  606,  605,
@@ -1167,14 +1167,14 @@ static yyconst flex_int16_t yy_chk[2368] =
       815,  815,  815,  815,  815,  815,  815,  815,  815,  815,
       815,  816,  816,  816,  816,  816,  816,  816,  816,  816,
       816,  816,  816,  817,  817,  817,  817,  817,  817,  817,
-      817,  817,  817,  817,  817,  819,  819,  819,  683,  819,
+      817,  817,  817,  817,  817,  819,  819,  819,  819,  683,
 
-      819,  819,  820,  820,  682,  820,  820,  821,  674,  821,
+      819,  819,  820,  820,  820,  682,  820,  821,  674,  821,
       821,  821,  821,  821,  821,  821,  821,  821,  821,  822,
       822,  822,  822,  822,  822,  822,  822,  822,  822,  822,
       822,  823,  823,  823,  823,  823,  823,  823,  823,  823,
-      823,  823,  823,  824,  824,  673,  824,  824,  825,  825,
-      666,  825,  825,  825,  826,  826,  665,  826,  826,  827,
+      823,  823,  823,  824,  824,  824,  673,  824,  825,  825,
+      666,  825,  825,  825,  826,  826,  826,  665,  826,  827,
       827,  664,  827,  827,  827,  828,  828,  828,  828,  828,
       828,  828,  828,  828,  828,  828,  828,  829,  829,  829,
       829,  829,  829,  829,  829,  829,  829,  829,  829,  830,
@@ -1183,13 +1183,13 @@ static yyconst flex_int16_t yy_chk[2368] =
       830,  831,  650,  649,  831,  831,  831,  831,  831,  831,
       831,  831,  831,  832,  832,  832,  832,  832,  832,  832,
       832,  832,  832,  832,  832,  833,  638,  637,  833,  833,
-      833,  833,  833,  833,  833,  833,  833,  834,  834,  632,
-      834,  834,  835,  835,  835,  631,  835,  835,  835,  836,
-      836,  630,  836,  840,  840,  629,  840,  840,  628,  627,
-      840,  842,  842,  620,  842,  842,  619,  611,  842,  844,
-      844,  610,  844,  844,  609,  607,  844,  845,  845,  606,
-      845,  845,  605,  586,  845,  846,  846,  585,  846,  846,
-      584,  583,  846,  582,  581,  574,  573,  572,  571,  570,
+      833,  833,  833,  833,  833,  833,  833,  834,  834,  834,
+      632,  834,  835,  835,  835,  835,  631,  835,  835,  836,
+      630,  836,  629,  836,  840,  840,  840,  628,  840,  627,
+      620,  840,  842,  842,  842,  619,  842,  611,  610,  842,
+      844,  844,  844,  609,  844,  607,  606,  844,  845,  845,
+      845,  605,  845,  586,  585,  845,  846,  846,  846,  584,
+      846,  583,  582,  846,  581,  574,  573,  572,  571,  570,
 
       569,  568,  567,  566,  565,  564,  563,  556,  555,  554,
       553,  552,  551,  541,  540,  539,  538,  536,  535,  534,
@@ -1323,7 +1323,7 @@ int errorParse;
 enum {
   errorNone = 0,
   errorUnexpectedInput,
-  errorUnsupportedFeature, 
+  errorUnsupportedFeature,
 };
 
 }
@@ -3148,14 +3148,14 @@ void lex_t::accept (int token, const char* sz)
 
 void lex_t::input (char *buf, int *result, int max_size)
 {
-    printf("lex_t::input %p %d\n", buf, max_size);
+    /* printf("lex_t::input %p %d\n", buf, max_size); */
     *result = fread(buf, 1, max_size, file);
-    printf("lex_t::input result %d\n", *result);
+    /* printf("lex_t::input result %d\n", *result); */
 }
 
 int lex_t::line ()const
 {
-    printf("lex_t::line\n");
+    /* printf("lex_t::line\n"); */
     return yyget_lineno (scanner);
 }
 
@@ -3164,12 +3164,12 @@ extern "C" {
 
     void parse (FILE* file, PyObject* data)
     {
-	printf("parse bbparseAlloc\n");
+        /* printf("parse bbparseAlloc\n"); */
         void* parser = bbparseAlloc (malloc);
         yyscan_t scanner;
         lex_t lex;
 
-	printf("parse yylex_init\n");
+        /* printf("parse yylex_init\n"); */
         yylex_init (&scanner);
 
         lex.parser = parser;
@@ -3177,18 +3177,18 @@ extern "C" {
         lex.file = file;
         lex.data = data;
         lex.parse = bbparse;
-	printf("parse yyset_extra\n");
+        /*printf("parse yyset_extra\n"); */
         yyset_extra (&lex, scanner);
 
-	printf("parse yylex\n");
+        /* printf("parse yylex\n"); */
         int result = yylex (scanner);
-	
-	printf("parse result %d\n", result);
+
+        /* printf("parse result %d\n", result); */
 
         lex.accept (0);
-	printf("parse lex.accept\n");
+        /* printf("parse lex.accept\n"); */
         bbparseTrace (NULL, NULL);
-	printf("parse bbparseTrace\n");
+        /* printf("parse bbparseTrace\n"); */
 
         if (result != T_EOF)
            printf ("premature end of file\n");
diff --git a/lib/bb/parse/parse_c/bitbakescanner.l b/lib/bb/parse/parse_c/bitbakescanner.l
index f69a7325c..667f26098 100644
--- a/lib/bb/parse/parse_c/bitbakescanner.l
+++ b/lib/bb/parse/parse_c/bitbakescanner.l
@@ -91,7 +91,7 @@ int errorParse;
 enum {
   errorNone = 0,
   errorUnexpectedInput,
-  errorUnsupportedFeature, 
+  errorUnsupportedFeature,
 };
 
 }
@@ -142,7 +142,7 @@ SSTRING         \'([^\n\r]|"\\\n")*\'
 VALUE           ([^'" \t\n])|([^'" \t\n]([^\n]|(\\\n))*[^'" \t\n])
 
 C_SS            [a-zA-Z_]
-C_SB            [a-zA-Z0-9_+-.]
+C_SB            [a-zA-Z0-9_+-./]
 REF             $\{{C_SS}{C_SB}*\}
 SYMBOL          {C_SS}{C_SB}*
 VARIABLE        $?{C_SS}({C_SB}*|{REF})*(\[[a-zA-Z0-9_]*\])?
@@ -265,14 +265,14 @@ void lex_t::accept (int token, const char* sz)
 
 void lex_t::input (char *buf, int *result, int max_size)
 {
-    printf("lex_t::input %p %d\n", buf, max_size);
+    /* printf("lex_t::input %p %d\n", buf, max_size); */
     *result = fread(buf, 1, max_size, file);
-    printf("lex_t::input result %d\n", *result);
+    /* printf("lex_t::input result %d\n", *result); */
 }
 
 int lex_t::line ()const
 {
-    printf("lex_t::line\n");
+    /* printf("lex_t::line\n"); */
     return yyget_lineno (scanner);
 }
 
@@ -281,12 +281,12 @@ extern "C" {
 
     void parse (FILE* file, PyObject* data)
     {
-	printf("parse bbparseAlloc\n");
+        /* printf("parse bbparseAlloc\n"); */
         void* parser = bbparseAlloc (malloc);
         yyscan_t scanner;
         lex_t lex;
 
-	printf("parse yylex_init\n");
+        /* printf("parse yylex_init\n"); */
         yylex_init (&scanner);
 
         lex.parser = parser;
@@ -294,18 +294,18 @@ extern "C" {
         lex.file = file;
         lex.data = data;
         lex.parse = bbparse;
-	printf("parse yyset_extra\n");
+        /*printf("parse yyset_extra\n"); */
         yyset_extra (&lex, scanner);
 
-	printf("parse yylex\n");
+        /* printf("parse yylex\n"); */
         int result = yylex (scanner);
-	
-	printf("parse result %d\n", result);
+
+        /* printf("parse result %d\n", result); */
 
         lex.accept (0);
-	printf("parse lex.accept\n");
+        /* printf("parse lex.accept\n"); */
         bbparseTrace (NULL, NULL);
-	printf("parse bbparseTrace\n");
+        /* printf("parse bbparseTrace\n"); */
 
         if (result != T_EOF)
            printf ("premature end of file\n");