langchain_recursive_character_text_splitter_chunking_quickstart.py

1from langchain_text_splitters import RecursiveCharacterTextSplitter
2
3# Load a long document to split
4with open("state_of_the_union.txt") as f:
5    state_of_the_union = f.read()
6
7# Initialize the text splitter with custom parameters
8text_splitter = RecursiveCharacterTextSplitter(
9    # Set a really small chunk size, just to show.
10    chunk_size=100,
11    chunk_overlap=20,
12    length_function=len,
13    is_separator_regex=False,
14)
15
16# Create documents from the text
17texts = text_splitter.create_documents([state_of_the_union])
18
19# Print the first two chunks
20print(texts[0])
21print(texts[1])