[SPARK-5968] [SQL] Suppresses ParquetOutputCommitter WARN logs
Please refer to the [JIRA ticket] [1] for the motivation. [1]: https://issues.apache.org/jira/browse/SPARK-5968 <!-- Reviewable:start --> [<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/4744) <!-- Reviewable:end --> Author: Cheng Lian <lian@databricks.com> Closes #4744 from liancheng/spark-5968 and squashes the following commits: caac6a8 [Cheng Lian] Suppresses ParquetOutputCommitter WARN logs
This commit is contained in:
parent
cf2e41653d
commit
8403331333
|
@ -18,11 +18,12 @@
|
||||||
package org.apache.spark.sql.parquet
|
package org.apache.spark.sql.parquet
|
||||||
|
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
|
import java.util.logging.Level
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration
|
import org.apache.hadoop.conf.Configuration
|
||||||
import org.apache.hadoop.fs.Path
|
import org.apache.hadoop.fs.Path
|
||||||
import org.apache.hadoop.fs.permission.FsAction
|
import org.apache.hadoop.fs.permission.FsAction
|
||||||
import parquet.hadoop.ParquetOutputFormat
|
import parquet.hadoop.{ParquetOutputCommitter, ParquetOutputFormat}
|
||||||
import parquet.hadoop.metadata.CompressionCodecName
|
import parquet.hadoop.metadata.CompressionCodecName
|
||||||
import parquet.schema.MessageType
|
import parquet.schema.MessageType
|
||||||
|
|
||||||
|
@ -99,7 +100,7 @@ private[sql] object ParquetRelation {
|
||||||
//
|
//
|
||||||
// Therefore we need to force the class to be loaded.
|
// Therefore we need to force the class to be loaded.
|
||||||
// This should really be resolved by Parquet.
|
// This should really be resolved by Parquet.
|
||||||
Class.forName(classOf[parquet.Log].getName())
|
Class.forName(classOf[parquet.Log].getName)
|
||||||
|
|
||||||
// Note: Logger.getLogger("parquet") has a default logger
|
// Note: Logger.getLogger("parquet") has a default logger
|
||||||
// that appends to Console which needs to be cleared.
|
// that appends to Console which needs to be cleared.
|
||||||
|
@ -108,6 +109,11 @@ private[sql] object ParquetRelation {
|
||||||
// TODO(witgo): Need to set the log level ?
|
// TODO(witgo): Need to set the log level ?
|
||||||
// if(parquetLogger.getLevel != null) parquetLogger.setLevel(null)
|
// if(parquetLogger.getLevel != null) parquetLogger.setLevel(null)
|
||||||
if (!parquetLogger.getUseParentHandlers) parquetLogger.setUseParentHandlers(true)
|
if (!parquetLogger.getUseParentHandlers) parquetLogger.setUseParentHandlers(true)
|
||||||
|
|
||||||
|
// Disables WARN log message in ParquetOutputCommitter.
|
||||||
|
// See https://issues.apache.org/jira/browse/SPARK-5968 for details
|
||||||
|
Class.forName(classOf[ParquetOutputCommitter].getName)
|
||||||
|
java.util.logging.Logger.getLogger(classOf[ParquetOutputCommitter].getName).setLevel(Level.OFF)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The element type for the RDDs that this relation maps to.
|
// The element type for the RDDs that this relation maps to.
|
||||||
|
|
Loading…
Reference in a new issue