2016-08-26 15:25:22 -04:00
|
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
|
|
<!--
|
|
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
|
|
~ this work for additional information regarding copyright ownership.
|
|
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
|
|
~ the License. You may obtain a copy of the License at
|
|
|
|
~
|
|
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
~
|
|
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
~ See the License for the specific language governing permissions and
|
|
|
|
~ limitations under the License.
|
|
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<parent>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
2018-11-14 19:22:23 -05:00
|
|
|
<artifactId>spark-parent_2.12</artifactId>
|
2021-08-31 13:04:14 -04:00
|
|
|
<version>3.2.1-SNAPSHOT</version>
|
2016-12-06 19:23:27 -05:00
|
|
|
<relativePath>../../pom.xml</relativePath>
|
2016-08-26 15:25:22 -04:00
|
|
|
</parent>
|
|
|
|
|
2018-11-14 19:22:23 -05:00
|
|
|
<artifactId>spark-mesos_2.12</artifactId>
|
2016-08-26 15:25:22 -04:00
|
|
|
<packaging>jar</packaging>
|
|
|
|
<name>Spark Project Mesos</name>
|
|
|
|
<properties>
|
|
|
|
<sbt.project.name>mesos</sbt.project.name>
|
[SPARK-18935][MESOS] Fix dynamic reservations on mesos
## What changes were proposed in this pull request?
- Solves the issue described in the ticket by preserving reservation and allocation info in all cases (port handling included).
- upgrades to 1.4
- Adds extra debug level logging to make debugging easier in the future, for example we add reservation info when applicable.
```
17/09/29 14:53:07 DEBUG MesosCoarseGrainedSchedulerBackend: Accepting offer: f20de49b-dee3-45dd-a3c1-73418b7de891-O32 with attributes: Map() allocation info: role: "spark-prive"
reservation info: name: "ports"
type: RANGES
ranges {
range {
begin: 31000
end: 32000
}
}
role: "spark-prive"
reservation {
principal: "test"
}
allocation_info {
role: "spark-prive"
}
```
- Some style cleanup.
## How was this patch tested?
Manually by running the example in the ticket with and without a principal. Specifically I tested it on a dc/os 1.10 cluster with 7 nodes and played with reservations. From the master node in order to reserve resources I executed:
```for i in 0 1 2 3 4 5 6
do
curl -i \
-d slaveId=90ec65ea-1f7b-479f-a824-35d2527d6d26-S$i \
-d resources='[
{
"name": "cpus",
"type": "SCALAR",
"scalar": { "value": 2 },
"role": "spark-role",
"reservation": {
"principal": ""
}
},
{
"name": "mem",
"type": "SCALAR",
"scalar": { "value": 8026 },
"role": "spark-role",
"reservation": {
"principal": ""
}
}
]' \
-X POST http://master.mesos:5050/master/reserve
done
```
Nodes had 4 cpus (m3.xlarge instances) and I reserved either 2 or 4 cpus (all for a role).
I verified it launches tasks on nodes with reserved resources under `spark-role` role only if
a) there are remaining resources for (*) default role and the spark driver has no role assigned to it.
b) the spark driver has a role assigned to it and it is the same role used in reservations.
I also tested this locally on my machine.
Author: Stavros Kontopoulos <st.kontopoulos@gmail.com>
Closes #19390 from skonto/fix_dynamic_reservation.
2017-11-29 17:15:35 -05:00
|
|
|
<mesos.version>1.4.0</mesos.version>
|
2016-08-26 15:25:22 -04:00
|
|
|
<mesos.classifier>shaded-protobuf</mesos.classifier>
|
|
|
|
</properties>
|
|
|
|
|
|
|
|
<dependencies>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
</dependency>
|
|
|
|
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
|
|
|
|
2017-02-08 07:21:49 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
|
|
|
|
2016-08-26 15:25:22 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.mesos</groupId>
|
|
|
|
<artifactId>mesos</artifactId>
|
|
|
|
<version>${mesos.version}</version>
|
|
|
|
<classifier>${mesos.classifier}</classifier>
|
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>com.google.protobuf</groupId>
|
|
|
|
<artifactId>protobuf-java</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
|
|
|
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.mockito</groupId>
|
|
|
|
<artifactId>mockito-core</artifactId>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
|
|
|
|
2017-08-17 18:47:07 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>${hive.group}</groupId>
|
|
|
|
<artifactId>hive-exec</artifactId>
|
2019-03-07 17:46:07 -05:00
|
|
|
<classifier>${hive.classifier}</classifier>
|
2017-08-17 18:47:07 -04:00
|
|
|
<scope>provided</scope>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>${hive.group}</groupId>
|
|
|
|
<artifactId>hive-metastore</artifactId>
|
|
|
|
<scope>provided</scope>
|
|
|
|
</dependency>
|
|
|
|
|
2016-08-26 15:25:22 -04:00
|
|
|
<!-- Explicitly depend on shaded dependencies from the parent, since shaded deps aren't transitive -->
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.google.guava</groupId>
|
|
|
|
<artifactId>guava</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-server</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-plus</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-util</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-http</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-servlet</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-servlets</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<!-- End of shaded deps. -->
|
|
|
|
|
|
|
|
</dependencies>
|
|
|
|
|
|
|
|
|
|
|
|
<build>
|
|
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
|
|
</build>
|
|
|
|
|
|
|
|
</project>
|