[Mumps2Py:] [36] added an extract_tokens function, mainly to handle variable renaming and initializing in tokprepass.py

[subversion@xxxxxxxxxxxxx]

Revision: 36
Author:   pgallot
Date:     2008-01-21 19:36:05 +0000 (Mon, 21 Jan 2008)

Log Message:
-----------
added an extract_tokens function, mainly to handle variable renaming and initializing in tokprepass.py

Modified Paths:
--------------
    trunk/mumps2py/tokens.py


Modified: trunk/mumps2py/tokens.py
===================================================================
--- trunk/mumps2py/tokens.py	2008-01-20 15:01:06 UTC (rev 35)
+++ trunk/mumps2py/tokens.py	2008-01-21 19:36:05 UTC (rev 36)
@@ -277,7 +277,37 @@
     FCC_CHARVAL:"asc",
     }
 
+def extract_subtokens(extract_dict, extract_types, token):
+    """extracts the subtokens of a given type(s)."""
+    if token.toktype in extract_types:
+        if extract_dict.has_key(token.toktype):
+            extract_dict[token.toktype].append(token)
+        else:
+            extract_dict[token.toktype] = [token, ]
 
+# NOTE: if I need to implement token tree traversal again, then I should
+# probably refactor it, so that it so that the tree traversing part calls
+# a callback with the current token and the args passed in.
+
+    for var in [key for key in token.__dict__.keys() if key \
+                not in ('start', 'end', 'toktype', 'lineno')]:
+
+        if isinstance(token.__dict__[var], DictType):
+            for (key, val) in token.__dict__[var].items():
+                if isinstance(val, Token):
+                    extract_subtokens(extract_dict, extract_types, val)
+        elif isinstance(token.__dict__[var], ListType):
+            for val in token.__dict__[var]:
+                if isinstance(val, DictType):
+                    for (key, dval) in val.items():
+                        if isinstance(dval, Token):
+                            extract_subtokens(extract_dict, extract_types, dval)
+                elif isinstance(val, Token):
+                    extract_subtokens(extract_dict, extract_types, val)
+        elif isinstance(token.__dict__[var], Token):
+            extract_subtokens(extract_dict, extract_types, token.__dict__[var])
+
+    
 def count_subtokens(count_dict, token):
     """counts the differen types of tokens and subtokens."""
     token_key = TOKEN_REVERSE_DICT.get(token.toktype, token.toktype)
@@ -412,3 +442,7 @@
     def count_subtokens(self, count_dict):
         """counts the different types of (sub)tokens including itself."""
         return count_subtokens(count_dict, self)
+
+    def extract_subtokens(self, extract_dict, extract_types):
+        """extract the (sub)tokens of the given types."""
+        return extract_subtokens(extract_dict, extract_types, self)