Did I find the right examples for you? yes no

All Samples(5)  |  Call(0)  |  Derive(0)  |  Import(5)
Module StdTokenizer

Implements the StdTokenizer class, which serves as a wrapper for Ping's
tokenizer in the Python distribution.

XXX While decoupling from Basil, I now assume this module can see every
submodule in the basil.common.python module.

$Id: StdTokenizer.py,v 1.2 2003/10/02 17:37:17 jriehl Exp $

src/m/y/mython-HEAD/basil/common/python/PgenParser.py   mython(Download)
def parseString (inString, tokenizer = None):
    if tokenizer == None:
        import StdTokenizer
        tokenizer = StdTokenizer.StdTokenizer().tokenizeString(inString)
    return handleStart(tokenizer)
def parseFile (filename, tokenizer = None):
    if tokenizer == None:
        import StdTokenizer
        tokenizer = StdTokenizer.StdTokenizer().tokenizeFile(filename)
    return handleStart(tokenizer)

src/m/y/mython-HEAD/basil/lang/python/DFAParser.py   mython(Download)
        from basil.lang.python import StdTokenizer
    except ImportError:
        import StdTokenizer
    if inputFile == None:
        inputFile = "<stdin>"

src/m/y/mython-HEAD/basil/common/python/PyPgen.py   mython(Download)
import PgenParser
import TokenUtils
import StdTokenizer
import token, string
 

src/m/y/mython-HEAD/basil/common/python/DFAParser.py   mython(Download)
        from basil.lang.python import StdTokenizer
    except ImportError:
        import StdTokenizer
    if inputFile == None:
        inputFile = "<stdin>"