Did I find the right examples for you? yes no Crawl my project Python Jobs
All Samples(17) | Call(16) | Derive(0) | Import(1)
def tokenize(text):
if text == '':
return []
result = []
index = 0
last_index = 0
text_token = ''
while True:
open_index = text.find('{', index)
if open_index == -1:
text_token = text[last_index:]
if text_token != '':
result.append(dict(type=TEXT_TOKEN, value=text_token))
break
next_char = text[open_index + 1]
if next_char in ['', ' ', '\t', '\n']:
index = open_index + 1
continue
index = open_index + 1
close_index = text.find('}', index)
if close_index == -1:
text_token = text[last_index:]
if text_token != '':
result.append(dict(type=TEXT_TOKEN, value=text_token))
break
text_token = text[last_index:open_index]
if text_token != '':
result.append(dict(type=TEXT_TOKEN, value=text_token))
name_token = text[index:close_index]
stripped_name_token = name_token.strip()
if stripped_name_token == '':
result.append(dict(type=TEXT_TOKEN,
value='{' + name_token + '}'))
else:
result.append(dict(type=NAME_TOKEN,
value=stripped_name_token))
index = close_index + 1
last_index = index
return result
from babelobviel.obvt import tokenize, NAME_TOKEN, TEXT_TOKEN
def test_tokenize_single_variable():
assert tokenize("{foo}") == [{'type': NAME_TOKEN,
'value': 'foo'}]
def test_tokenize_variable_in_text():
assert tokenize('the {foo} is great') == [
def test_variable_starts_text():
assert tokenize('{foo} is great') == [
{'type': NAME_TOKEN,
'value': 'foo'},
{'type': TEXT_TOKEN,
'value': ' is great'}]
def test_variable_ends_text():
assert tokenize('great is {foo}') == [