[Spark] 스파크 데이터프레임에 필터를 적용해 원하는 행을 추출하는 방법

코드

from pyspark.sql import SparkSession
from pyspark.sql import Row

spark = SparkSession\
        .builder\
        .appName("1_test_dataframe")\
        .getOrCreate()

sc = spark.sparkContext

data = [Row(name = 'a', age = 12, type = 'A', score = 90, year = 2012),
        Row(name = 'a', age = 15, type = 'B', score = 80, year = 2013),
        Row(name = 'b', age = 15, type = 'B', score = 80, year = 2014),
        Row(name = 'b', age = 21, type = 'F', score = 50, year = 2015),
        Row(name = 'c', age = 15, type = 'C', score = 70, year = 2016),
        Row(name = 'c', age = 33, type = 'F', score = 50, year = 2017)]

spark_df = sc.parallelize(data).toDF()

spark_df = spark_df.filter(spark_df.age > 20)

spark_df.show()

spark.stop()

결과