blingfire_text_tokenization_and_sentence_splitting_quickstart.py

1import blingfire as bf
2
3# Input text
4text = "After a billion-dollar lawsuit, the company was forced to change its name. This happened in 2021."
5
6# 1. Tokenization
7# text_to_words returns a space-separated string of tokens
8tokens = bf.text_to_words(text)
9print("Tokens:")
10print(tokens)
11
12# 2. Sentence Splitting
13# text_to_sentences returns a string with sentences separated by newlines
14sentences = bf.text_to_sentences(text)
15print("\nSentences:")
16print(sentences)
17
18# 3. Getting offsets (start and end positions of tokens)
19offsets = bf.text_to_words_with_offsets(text)
20print("\nTokens with Offsets:")
21print(offsets)