langchain_recursive_character_text_splitter_with_overlap.py

1from langchain_text_splitters import RecursiveCharacterTextSplitter
2
3# The text to be chunked
4text_content = """
5Your long document text goes here. 
6Standard chunking usually involves breaking text down by characters 
7while maintaining a specific chunk size and overlap to preserve context.
8"""
9
10# Initialize the splitter
11# Standard defaults are often 1000 characters with 200 character overlap
12text_splitter = RecursiveCharacterTextSplitter(
13    chunk_size=1000,
14    chunk_overlap=200,
15    length_function=len,
16    is_separator_regex=False,
17)
18
19# Create the chunks
20chunks = text_splitter.create_documents([text_content])
21
22# Output the result
23for i, chunk in enumerate(chunks):
24    print(f"Chunk {i+1}:\n{chunk.page_content}\n")