[Spark] RDD에서 filter 적용해서 데이터 처리하는 방법

코드

from pyspark.sql import SparkSession


if __name__ == "__main__":

    spark = SparkSession\
            .builder\
            .appName("0_save_file")\
            .getOrCreate()

    sc = spark.sparkContext

    line_1 = sc.parallelize(['0', '1', '2', '3', '4'])
    line_2 = sc.parallelize(['5', '6', '7', '8', '9'])
    line_3 = sc.parallelize(['10', '11', '12', '13', '14'])

    line_all = line_1.union(line_2).union(line_3)

    line_filter = line_all.filter(lambda x: "1" in x)

    print('line_all    :', f'{line_all.count()}, {line_all.collect()}')
    print('list_filter :', f'{line_filter.count()}, {line_filter.collect()}')

    spark.stop()

결과