Did I find the right examples for you? yes no

All Samples(1)  |  Call(0)  |  Derive(0)  |  Import(1)

src/s/t/stav-HEAD/server/src/tokenise.py   stav(Download)
 
try:
    from config import (TOKENIZATION, WHITESPACE_TOKENIZATION,
            PTBLIKE_TOKENIZATION, JAPANESE_TOKENIZATION)
except ImportError:
def jp_token_boundary_gen(text):
    # TODO: consider honoring WHITESPACE_TOKENIZATION for japanese also
    if TOKENIZATION is not None and TOKENIZATION != JAPANESE_TOKENIZATION:
        from message import Messager
        Messager.warning('Ignoring unexpected TOKENIZATION '