pyspark_pandas_dataframe_creation_viewing_grouping_quickstart.py

1import pandas as pd
2import numpy as np
3import pyspark.pandas as ps
4from pyspark.sql import SparkSession
5
6# Object Creation
7# Creating a pandas-on-Spark Series by passing a list of values
8s = ps.Series([1, 3, 5, np.nan, 6, 8])
9
10# Creating a pandas-on-Spark DataFrame by passing a dict of objects
11pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
12                    'b': [100, 200, 300, 400, 500, 600],
13                    'c': ["one", "two", "three", "four", "five", "six"]},
14                   index=[10, 20, 30, 40, 50, 60])
15psdf = ps.from_pandas(pdf)
16
17# Viewing Data
18# Display the top rows of the frame
19print(psdf.head())
20
21# Describe shows a quick statistic summary of your data
22print(psdf.describe())
23
24# Sorting by index or values
25print(psdf.sort_index(ascending=False))
26print(psdf.sort_values(by='b'))
27
28# Selection
29# Selecting a single column
30print(psdf['a'])
31
32# Selecting by label
33print(psdf.loc[10:20])
34
35# Grouping
36# Grouping and then applying the sum() function to the resulting groups
37print(psdf.groupby('c').sum())
38
39# Plotting
40# Note: This requires a visualization library like matplotlib installed
41# psdf.plot.area()