8ef4da753d
## What changes were proposed in this pull request? Fixed the `spark-<version>-yarn-shuffle.jar` artifact packaging to shade the native netty libraries: - shade the `META-INF/native/libnetty_*` native libraries when packagin the yarn shuffle service jar. This is required as netty library loader derives that based on shaded package name. - updated the `org/spark_project` shade package prefix to `org/sparkproject` (i.e. removed underscore) as the former breaks the netty native lib loading. This was causing the yarn external shuffle service to fail when spark.shuffle.io.mode=EPOLL ## How was this patch tested? Manual tests Closes #24502 from amuraru/SPARK-27610_master. Authored-by: Adi Muraru <amuraru@adobe.com> Signed-off-by: Marcelo Vanzin <vanzin@cloudera.com>
41 lines
2 KiB
Plaintext
41 lines
2 KiB
Plaintext
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# Set everything to be logged to the console
|
|
log4j.rootCategory=INFO, console
|
|
log4j.appender.console=org.apache.log4j.ConsoleAppender
|
|
log4j.appender.console.target=System.err
|
|
log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
|
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
|
|
|
|
# Set the default spark-shell log level to WARN. When running the spark-shell, the
|
|
# log level for this class is used to overwrite the root logger's log level, so that
|
|
# the user can have different defaults for the shell and regular Spark apps.
|
|
log4j.logger.org.apache.spark.repl.Main=WARN
|
|
|
|
# Settings to quiet third party logs that are too verbose
|
|
log4j.logger.org.sparkproject.jetty=WARN
|
|
log4j.logger.org.sparkproject.jetty.util.component.AbstractLifeCycle=ERROR
|
|
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
|
|
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
|
|
log4j.logger.org.apache.parquet=ERROR
|
|
log4j.logger.parquet=ERROR
|
|
|
|
# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
|
|
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
|
|
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
|