json_stream_rs_tokenizer_python_quickstart_example.py

1import io
2from json_stream_rs_tokenizer import PyTokenizer
3
4# A sample JSON stream (can be a file-like object)
5f = io.BytesIO(b'{"a": 1, "b": [2, 3]}')
6
7# Initialize the tokenizer
8tokenizer = PyTokenizer(f)
9
10# Iterate over tokens
11for token in tokenizer:
12    print(token)