sqlglotrs_rust_tokenizer_quickstart_for_faster_sql_parsing.py

1import sqlglot
2from sqlglotrs import Tokenizer
3
4# sqlglotrs is a Rust port of the sqlglot tokenizer.
5# It is designed to be a drop-in replacement for the standard Python tokenizer
6# to significantly speed up the parsing process.
7
8# To use it, you simply assign the sqlglotrs Tokenizer to the desired dialect.
9sqlglot.Dialect.get_or_raise("duckdb").Tokenizer = Tokenizer
10
11# Now, calls to transpile or parse will use the Rust-backed tokenizer
12sql = "SELECT * FROM table WHERE x = 1"
13transpiled = sqlglot.transpile(sql, read="duckdb", write="postgres")[0]
14
15print(transpiled)