[Spark] 10개의 숫자를 map 하고, reduce 처리 하는 방법

코드

from pyspark.sql import SparkSession

spark = SparkSession\
        .builder\
        .appName("0_save_file")\
        .getOrCreate()

sc = spark.sparkContext

data = sc.parallelize(range(0, 10))
data_map = data.map(lambda x: x * x)

data_reduce = data.reduce(lambda x, y: x + y)
data_map_reduce = data_map.reduce(lambda x, y: x + y)

print(f'data            : {data.collect()}')
print(f'data_map        : {data_map.collect()}')
print(f'data_reduce     : {data_reduce}')
print(f'data_map_reduce : {data_map_reduce}')

spark.stop()

결과