tiktoken_text_to_tokens_encoding_and_decoding.py

1import tiktoken
2
3# Load an encoding for a specific model
4encoding = tiktoken.encoding_for_model("gpt-4o")
5
6# Turn text into tokens
7tokens = encoding.encode("tiktoken is great!")
8print(f"Tokens: {tokens}")
9
10# Turn tokens back into text
11text = encoding.decode(tokens)
12print(f"Text: {text}")