From f9efdeea8c1b7328d941b7e3d894a22afd3a42a2 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apple.com>
Date: Mon, 27 Sep 2021 15:17:04 +0800
Subject: [PATCH] [SPARK-36835][FOLLOWUP][BUILD][TEST-HADOOP2.7] Move
 spark.yarn.isHadoopProvided to parent pom

### What changes were proposed in this pull request?

Move `spark.yarn.isHadoopProvided` to Spark parent pom, so that under `resource-managers/yarn` we can make `hadoop-3.2` as the default profile.

### Why are the changes needed?

Currently under `resource-managers/yarn` there are 3 maven profiles : `hadoop-provided`, `hadoop-2.7`, and `hadoop-3.2`, of which `hadoop-3.2` is activated by default (via `activeByDefault`). The activation, however, doesn't work when there is other explicitly activated profiles. In specific, if users build Spark with `hadoop-provided`, maven will fail because it can't find Hadoop 3.2 related dependencies, which are defined in the `hadoop-3.2` profile section.

To fix the issue, this proposes to move the `hadoop-provided` section to the parent pom. Currently this is only used to define a property `spark.yarn.isHadoopProvided`, and it shouldn't matter where we define it.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Tested via running the command:
```
build/mvn clean package -DskipTests -B -Pmesos -Pyarn -Pkubernetes -Pscala-2.12 -Phadoop-provided
```
which was failing before this PR but is succeeding with it.

Also checked active profiles with the command:
```
build/mvn -Pyarn -Phadoop-provided help:active-profiles
```
and it shows that `hadoop-3.2` is active for `spark-yarn` module now.

Closes #34110 from sunchao/SPARK-36835-followup2.

Authored-by: Chao Sun <sunchao@apple.com>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
---
 pom.xml                        | 4 ++++
 resource-managers/yarn/pom.xml | 7 -------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/pom.xml b/pom.xml
index e7ca906a9f..72410ecfe8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -269,6 +269,7 @@
     <hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
     <hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
     <hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
+    <spark.yarn.isHadoopProvided>false</spark.yarn.isHadoopProvided>
 
     <!--
       Overridable test home. So that you can call individual pom files directly without
@@ -3485,6 +3486,9 @@
     -->
     <profile>
       <id>hadoop-provided</id>
+      <properties>
+        <spark.yarn.isHadoopProvided>true</spark.yarn.isHadoopProvided>
+      </properties>
     </profile>
     <profile>
       <id>hive-provided</id>
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 12a600fa0e..287f636b9e 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -30,16 +30,9 @@
   <properties>
     <sbt.project.name>yarn</sbt.project.name>
     <jersey-1.version>1.19</jersey-1.version>
-    <spark.yarn.isHadoopProvided>false</spark.yarn.isHadoopProvided>
   </properties>
 
   <profiles>
-    <profile>
-      <id>hadoop-provided</id>
-      <properties>
-        <spark.yarn.isHadoopProvided>true</spark.yarn.isHadoopProvided>
-      </properties>
-    </profile>
     <profile>
       <id>hadoop-2.7</id>
       <dependencies>