dbt_athena_python_model_spark_dataframe_transformation.py

1import pandas as pd
2
3def model(dbt, sessions):
4    # dbt configuration
5    dbt.config(
6        materialized="table",
7    )
8
9    # Reading data from a dbt model or source
10    # This returns a Spark DataFrame
11    df = dbt.ref("my_sql_model")
12
13    # You can perform transformations using the PySpark API
14    df_transformed = df.filter(df["column_name"] > 0)
15
16    # The function must return a Spark DataFrame or a Pandas DataFrame
17    return df_transformed