awswrangler_pandas_to_s3_parquet_with_glue_catalog.py

1import awswrangler as wr
2import pandas as pd
3
4# Creating the DataFrame
5df = pd.DataFrame({"id": [1, 2], "value": ["foo", "bar"]})
6
7# Writing to Amazon S3
8wr.s3.to_parquet(
9    df=df,
10    path="s3://bucket/dataset/",
11    dataset=True,
12    database="my_db",  # Optional: To create/update a table in the Glue Catalog
13    table="my_table"
14)
15
16# Reading from Amazon S3
17df2 = wr.s3.read_parquet(path="s3://bucket/dataset/", dataset=True)
18
19# Reading via Amazon Athena
20df3 = wr.athena.read_sql_query("SELECT * FROM my_table", database="my_db")