[Spark] To set up memory in a spark session

Code

from pyspark.sql import SparkSession

spark = SparkSession.builder \
                    .appName('3_test_sparksession') \
                    .master('spark://spark-master:17077') \
                    .config('spark.driver.cores', '1') \
                    .config('spark.driver.memory','1g') \
                    .config('spark.executor.memory', '1g') \
                    .config('spark.executor.cores', '2') \
                    .config('spark.cores.max', '2') \
                    .getOrCreate()

sc = spark.sparkContext

for setting in sc._conf.getAll():
    print(setting)

sc.stop()

 

Result