[Spark] RDD를 필터처리하는 방법

코드

from pyspark.sql import SparkSession

if __name__ == "__main__":

    spark = SparkSession\
            .builder\
            .appName("0_save_file")\
            .getOrCreate()

    sc = spark.sparkContext

    lines = sc.parallelize(['i love you', 'you are my friend', 'my name is park'])
    
    print(lines.collect())

    pairs = lines.map(lambda s: s.split(" "))
    pairs = pairs.filter(lambda x: len(x) > 3)

    print(pairs.collect())

    spark.stop()

결과