dbt_athena_python_model_with_pandas_transformation.py

1import pandas as pd
2
3def model(dbt, session):
4    # dbt configuration
5    dbt.config(
6        materialized="table",
7        table_type="iceberg",
8        location_property="external_location"
9    )
10
11    # Reading data from an upstream dbt model or source
12    # The 'session' object is an AWS Glue PySpark/Python session
13    df = dbt.ref("my_upstream_sql_model").to_pandas()
14
15    # Simple transformation logic
16    df["transformed_column"] = df["existing_column"].apply(lambda x: x.upper() if x else x)
17
18    # Return the final dataframe to be written to S3 and cataloged in Glue/Athena
19    return df