livekit_blingfire_streaming_text_sentence_tokenizer.py

1from livekit.plugins import blingfire
2
3def main():
4    # Initialize the Blingfire sentence tokenizer
5    tokenizer = blingfire.SentenceTokenizer()
6
7    # Create a text stream to simulate incoming data
8    text_stream = tokenizer.stream()
9
10    # Push chunks of text into the tokenizer
11    text_stream.push("Hello world. This is a ")
12    text_stream.push("test of the Blingfire tokenizer.")
13    text_stream.flush()
14
15    # Iterate through the segmented sentences
16    for sentence in text_stream:
17        print(f"Segmented sentence: {sentence}")
18
19if __name__ == "__main__":
20    main()