dbt_spark_python_model_with_ref_and_filter_transform.py

1import pandas as pd
2
3def model(dbt, session):
4    # dbt configuration
5    dbt.config(materialized="table")
6
7    # Define the data source
8    # 'my_upstream_model' is a reference to another dbt model or seed
9    df = dbt.ref("my_upstream_model")
10
11    # Perform a simple transformation
12    # When using dbt-spark, 'df' is a PySpark DataFrame
13    final_df = df.filter(df.id > 0)
14
15    # Return the final DataFrame to be materialized
16    return final_df