python_tokenize_untokenize_source_code_reconstruction.py

1import tokenize
2import io
3
4# Sample code to tokenize and then untokenize
5source_code = "def add(a, b): return a + b"
6
7# Tokenize the source code
8tokens = tokenize.tokenize(io.BytesIO(source_code.encode('utf-8')).readline)
9
10# Untokenize back into source code
11reconstructed_code = tokenize.untokenize(tokens).decode('utf-8')
12
13print(reconstructed_code)