tokenize_rt_python_source_roundtrip_to_tokens_and_back.py

1from tokenize_rt import src_to_tokens
2from tokenize_rt import tokens_to_src
3
4def main():
5    code = "x = 1  # some comment\n"
6    
7    # Convert source code to tokens
8    tokens = src_to_tokens(code)
9    print(f"Tokens: {tokens}")
10    
11    # Convert tokens back to source (round-trip)
12    restored_code = tokens_to_src(tokens)
13    print(f"Restored code: {restored_code}")
14    
15    assert code == restored_code
16
17if __name__ == "__main__":
18    main()