spark 开启历史日志监控
vi spark-defaults.conf
spark.eventLog.enabled true
spark.eventLog.dir hdfs://webcluster/directory
spark.eventLog.compress true
vi spark-env.sh
export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=7777 -Dspark.history.retainedApplications=10 -Dspark.history.fs.logDirectory=hdfs://webcluster/directory"
spark.eventLog.enabled true
spark.eventLog.dir hdfs://webcluster/directory
spark.eventLog.compress true
vi spark-env.sh
export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=7777 -Dspark.history.retainedApplications=10 -Dspark.history.fs.logDirectory=hdfs://webcluster/directory"
启动 start-history-server.sh
./spark-sql执行相关sql:
针对thrift server必须重启后生效.