[Spark] 스파크 데이터프레임을 agg() 이용해서 groupBy() 하는 방법

코드

from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.sql.functions import max, avg, sum, min

spark = SparkSession\
        .builder\
        .appName("1_test_dataframe")\
        .getOrCreate()

sc = spark.sparkContext

data = [Row(name = 'a', age = 12, type = 'A', score = 90, year = 2012),
        Row(name = 'a', age = 15, type = 'B', score = 80, year = 2013),
        Row(name = 'b', age = 15, type = 'B', score = 80, year = 2014),
        Row(name = 'b', age = 21, type = 'F', score = 50, year = 2015),
        Row(name = 'c', age = 15, type = 'C', score = 70, year = 2016),
        Row(name = 'c', age = 33, type = 'F', score = 50, year = 2017)]

spark_df = sc.parallelize(data).toDF()

spark_df = spark_df.groupBy('name').agg(sum('age'), sum('score'), sum('year')).select('*')

spark_df.show()

spark.stop()

결과