standard_chunk_text_splitting_with_max_token_size.py

1from standard_chunk import chunk_text
2
3# The text you want to process
4text = """
5Standard Chunk is a lightweight library designed to provide consistent, 
6high-quality text chunking for LLM applications. It ensures that 
7your context windows are utilized efficiently by breaking down 
8large documents into smaller, meaningful pieces.
9"""
10
11# Define the maximum number of tokens per chunk
12max_tokens = 50
13
14# Split the text into chunks
15chunks = chunk_text(text, max_tokens=max_tokens)
16
17# Display the results
18for i, chunk in enumerate(chunks):
19    print(f"Chunk {i+1}:")
20    print(chunk)
21    print("-" * 20)