semchunk_text_chunking_with_tiktoken_tokenizer.py

1import semchunk
2import tiktoken
3
4# 1. Initialize a tokenizer (semchunk works with any tokenizer)
5tokenizer = tiktoken.encoding_for_model('gpt-4')
6
7# 2. Define the text to be chunked
8text = 'The quick brown fox jumps over the lazy dog.'
9
10# 3. Create a chunker function by passing the tokenizer and maximum chunk size
11# semchunk.chunker() returns a function that can be used to chunk text
12chunker = semchunk.chunker(tokenizer, max_token_chars=None, memoize=True)
13
14# 4. Chunk the text into segments of at most 5 tokens
15chunks = chunker(text, 5)
16
17# 5. Display the resulting chunks
18for chunk in chunks:
19    print(f"'{chunk}'")