[Spark] 스파크 데이터프레임 groupBy 사용하는 방법

코드

from pyspark.sql import SparkSession
from pyspark.sql import Row

spark = SparkSession\
        .builder\
        .appName("1_test_dataframe")\
        .getOrCreate()

sc = spark.sparkContext

data = [Row(name = 'a', age = 12, type = 'A', score = 90, year = 2012),
        Row(name = 'a', age = 15, type = 'B', score = 80, year = 2013),
        Row(name = 'b', age = 15, type = 'B', score = 80, year = 2014),
        Row(name = 'b', age = 21, type = 'F', score = 50, year = 2015),
        Row(name = 'c', age = 15, type = 'C', score = 70, year = 2016),
        Row(name = 'c', age = 33, type = 'F', score = 50, year = 2017)]

spark_df = sc.parallelize(data).toDF()

spark_df = spark_df.groupBy('name').sum('score').select('*')

spark_df.show()

spark.stop()

결과