pbspark_protobuf_binary_to_spark_dataframe_conversion.py

1from pyspark.sql import SparkSession
2from pbspark import MessageConverter
3from person_pb2 import Person # Compiled protobuf class
4
5spark = SparkSession.builder.getOrCreate()
6
7# Create a converter for the Person message
8mc = MessageConverter()
9
10# Sample data: list of serialized Protobuf messages
11data = [
12    Person(name="John Doe", id=123, email="john@example.com").SerializeToString(),
13    Person(name="Jane Doe", id=456).SerializeToString(),
14]
15
16# Create a Spark DataFrame with a single binary column
17df = spark.createDataFrame(data, "binary").toDF("value")
18
19# Use pbspark to decode the binary column into a structured format
20# from_protobuf returns a column with the schema of the Protobuf message
21from pbspark import from_protobuf
22decoded_df = df.select(from_protobuf(df.value, Person).alias("person"))
23
24# Show the results
25decoded_df.select("person.*").show()