diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/examples/ReadORCFile.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/examples/ReadORCFile.scala index da94cc9..946c75c 100644 --- a/src/main/scala/com/sparkbyexamples/spark/dataframe/examples/ReadORCFile.scala +++ b/src/main/scala/com/sparkbyexamples/spark/dataframe/examples/ReadORCFile.scala @@ -21,7 +21,7 @@ object ReadORCFile extends App{ .orc("/tmp/orc/data.orc") df.write.mode("overwrite") - .option("compression","none12") + .option("compression","none") .orc("/tmp/orc/data-nocomp.orc") df.write.mode("overwrite") diff --git a/src/main/scala/com/sparkbyexamples/spark/spark30/ADQExample.scala b/src/main/scala/com/sparkbyexamples/spark/spark30/AQEExample.scala similarity index 72% rename from src/main/scala/com/sparkbyexamples/spark/spark30/ADQExample.scala rename to src/main/scala/com/sparkbyexamples/spark/spark30/AQEExample.scala index 44910c2..d89670b 100644 --- a/src/main/scala/com/sparkbyexamples/spark/spark30/ADQExample.scala +++ b/src/main/scala/com/sparkbyexamples/spark/spark30/AQEExample.scala @@ -2,7 +2,7 @@ package com.sparkbyexamples.spark.spark30 import org.apache.spark.sql.SparkSession -object ADQExample extends App{ +object AQEExample extends App{ val spark: SparkSession = SparkSession.builder() .master("local[5]") @@ -25,12 +25,11 @@ object ADQExample extends App{ val df = simpleData.toDF("employee_name","department","state","salary","age","bonus") val df1=df.groupBy("department").count() - println(df1.rdd.getNumPartitions) - - spark.conf.set("spark.sql.adaptive.enabled",200) - val df2=df.groupBy("department").count() - println(df2.rdd.getNumPartitions) + println(s"Number of partitions in RDD (AQE=[${spark.conf.get("spark.sql.adaptive.enabled")}]) = [${df1.rdd.getNumPartitions}]") + spark.conf.set("spark.sql.adaptive.enabled", true) + val df2=df.groupBy("department").count() + println(s"Number of partitions in RDD (AQE=[${spark.conf.get("spark.sql.adaptive.enabled")}]) = [${df2.rdd.getNumPartitions}]") }