Did I find the right examples for you? yes no

All Samples(1)  |  Call(0)  |  Derive(0)  |  Import(1)

src/s/t/stav-HEAD/server/src/tokenise.py   stav(Download)
 
try:
    from config import (TOKENIZATION, WHITESPACE_TOKENIZATION,
            PTBLIKE_TOKENIZATION, JAPANESE_TOKENIZATION)
except ImportError:
def en_token_boundary_gen(text):
    if TOKENIZATION is None or TOKENIZATION == WHITESPACE_TOKENIZATION:
        return en_token_boundary_gen_simple(text)        
    elif TOKENIZATION == PTBLIKE_TOKENIZATION:
        return en_token_boundary_gen_gtb(text)