awswrangler_pandas_dataframe_to_s3_parquet_with_glue_catalog.py

1import awswrangler as wr
2import pandas as pd
3
4# Creating the DataFrame
5df = pd.DataFrame({"id": [1, 2], "value": ["foo", "bar"]})
6
7# Writing to S3
8wr.s3.to_parquet(
9    df=df,
10    path="s3://bucket/dataset/",
11    dataset=True,
12    database="my_db",  # Optional: To create/update a Glue Catalog table
13    table="my_table"
14)
15
16# Reading from S3
17df_new = wr.s3.read_parquet(path="s3://bucket/dataset/", dataset=True)