Did I find the right examples for you? yes no      Crawl my project      Python Jobs

All Samples(17)  |  Call(16)  |  Derive(0)  |  Import(1)

        def tokenize(text):
    if text == '':
        return []
    result = []
    index = 0
    last_index = 0
    text_token = ''
    while True:
        open_index = text.find('{', index)
        if open_index == -1:
            text_token = text[last_index:]
            if text_token != '':
                result.append(dict(type=TEXT_TOKEN, value=text_token))
            break
        next_char = text[open_index + 1]
        if next_char in ['', ' ', '\t', '\n']:
            index = open_index + 1
            continue
        index = open_index + 1
        close_index = text.find('}', index)
        if close_index == -1:
            text_token = text[last_index:]
            if text_token != '':
                result.append(dict(type=TEXT_TOKEN, value=text_token))
            break
        text_token = text[last_index:open_index]
        if text_token != '':
            result.append(dict(type=TEXT_TOKEN, value=text_token))
        name_token = text[index:close_index]
        stripped_name_token = name_token.strip()
        if stripped_name_token == '':
            result.append(dict(type=TEXT_TOKEN,
                               value='{' + name_token + '}'))
        else:
            result.append(dict(type=NAME_TOKEN,
                               value=stripped_name_token))
        index = close_index + 1
        last_index = index
    return result
        


src/b/a/babel-obviel-0.7.1/babelobviel/test/test_tokenize.py   babel-obviel(Download)
from babelobviel.obvt import tokenize, NAME_TOKEN, TEXT_TOKEN
 
def test_tokenize_single_variable():
    assert tokenize("{foo}") == [{'type': NAME_TOKEN,
                                  'value': 'foo'}]
 
def test_tokenize_variable_in_text():
    assert tokenize('the {foo} is great') == [
def test_variable_starts_text():
    assert tokenize('{foo} is great') == [
        {'type': NAME_TOKEN,
         'value': 'foo'},
        {'type': TEXT_TOKEN,
         'value': ' is great'}]
 
def test_variable_ends_text():
    assert tokenize('great is {foo}') == [