quinn_validate_presence_pyspark_dataframe_column_check.py

1from pyspark.sql import SparkSession
2import quinn
3
4spark = SparkSession.builder.master("local").appName("quinn-example").getOrCreate()
5
6data = [("jose", 1), ("li", 2), ("sam", 3)]
7source_df = spark.createDataFrame(data, ["name", "age"])
8
9# This will pass because "name" and "age" columns are present
10quinn.validate_presence(source_df, ["name", "age"])
11
12# This would raise an error because "city" is missing
13# quinn.validate_presence(source_df, ["city"])
14
15source_df.show()