Did I find the right examples for you? yes no      Crawl my project      Python Jobs

All Samples(54)  |  Call(27)  |  Derive(0)  |  Import(27)
Tokenize a JavaScript source.  Returns a generator of tokens.
    

        def tokenize(source):
    """Tokenize a JavaScript source.  Returns a generator of tokens.
    """
    may_divide = False
    pos = 0
    lineno = 1
    end = len(source)

    while pos < end:
        # handle regular rules first
        for token_type, rule in rules:
            match = rule.match(source, pos)
            if match is not None:
                break
        # if we don't have a match we don't give up yet, but check for
        # division operators or regular expression literals, based on
        # the status of `may_divide` which is determined by the last
        # processed non-whitespace token using `indicates_division`.
        else:
            if may_divide:
                match = division_re.match(source, pos)
                token_type = 'operator'
            else:
                match = regex_re.match(source, pos)
                token_type = 'regexp'
            if match is None:
                # woops. invalid syntax. jump one char ahead and try again.
                pos += 1
                continue

        token_value = match.group()
        if token_type is not None:
            token = Token(token_type, token_value, lineno)
            may_divide = indicates_division(token)
            yield token
        lineno += len(line_re.findall(token_value))
        pos = match.end()
        


src/h/u/hue-HEAD/desktop/core/ext-py/Babel-0.9.6/babel/messages/extract.py   hue(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/s/o/socorro-HEAD/webapp-django/vendor/lib/python/babel/messages/extract.py   socorro(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/m/i/microblog-HEAD/flask/lib/python2.7/site-packages/babel/messages/extract.py   microblog(Download)
    :param options: a dictionary of additional options (optional)
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/h/o/hortonworks-sandbox-HEAD/desktop/core/ext-py/Babel-0.9.6/babel/messages/extract.py   hortonworks-sandbox(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/f/j/fjord-HEAD/vendor/packages/Babel/babel/messages/extract.py   fjord(Download)
    :param options: a dictionary of additional options (optional)
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/k/i/kitsune-HEAD/vendor/packages/Babel/babel/messages/extract.py   kitsune(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/g/l/glow-HEAD/vendor/lib/python/babel/messages/extract.py   glow(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/z/e/ZenPacks.zenoss.OpenStack-HEAD/src/Babel-1.3/babel/messages/extract.py   ZenPacks.zenoss.OpenStack(Download)
    :param options: a dictionary of additional options (optional)
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/g/a/gae-boilerplate-HEAD/bp_includes/external/babel/messages/extract.py   gae-boilerplate(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

src/w/e/webapp-improved-HEAD/lib/babel/babel/messages/extract.py   webapp-improved(Download)
    :rtype: ``iterator``
    """
    from babel.messages.jslexer import tokenize, unquote_string
    funcname = message_lineno = None
    messages = []
    call_stack = -1
 
    for token in tokenize(fileobj.read().decode(encoding)):
        if token.type == 'operator' and token.value == '(':
            if funcname:

  1 | 2 | 3  Next