blingfire_text_tokenization_words_and_sentences.py

1from blingfire import text_to_words, text_to_sentences
2
3s = "This is the BlingFire tokenizer. It is fast!"
4
5# Tokenize into words
6words = text_to_words(s)
7print(words)
8
9# Tokenize into sentences
10sentences = text_to_sentences(s)
11print(sentences)