2016-01-15 20:07:24 -05:00
|
|
|
JavaEWAH-0.3.2.jar
|
|
|
|
RoaringBitmap-0.5.11.jar
|
|
|
|
ST4-4.0.4.jar
|
|
|
|
activation-1.1.1.jar
|
2017-10-19 01:30:55 -04:00
|
|
|
aircompressor-0.8.jar
|
2016-03-31 12:25:09 -04:00
|
|
|
antlr-2.7.7.jar
|
|
|
|
antlr-runtime-3.4.jar
|
2017-08-24 19:33:55 -04:00
|
|
|
antlr4-runtime-4.7.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
aopalliance-1.0.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
aopalliance-repackaged-2.4.0-b34.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
apache-log4j-extras-1.2.17.jar
|
|
|
|
apacheds-i18n-2.0.0-M15.jar
|
|
|
|
apacheds-kerberos-codec-2.0.0-M15.jar
|
|
|
|
api-asn1-api-1.0.0-M20.jar
|
|
|
|
api-util-1.0.0-M20.jar
|
|
|
|
arpack_combined_all-0.1.jar
|
2017-12-21 06:43:56 -05:00
|
|
|
arrow-format-0.8.0.jar
|
|
|
|
arrow-memory-0.8.0.jar
|
|
|
|
arrow-vector-0.8.0.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
automaton-1.11-8.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
avro-1.7.7.jar
|
|
|
|
avro-ipc-1.7.7.jar
|
|
|
|
avro-mapred-1.7.7-hadoop2.jar
|
|
|
|
base64-2.3.8.jar
|
2017-12-02 08:37:02 -05:00
|
|
|
bcprov-jdk15on-1.58.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
bonecp-0.8.0.RELEASE.jar
|
2017-08-09 02:44:10 -04:00
|
|
|
breeze-macros_2.11-0.13.2.jar
|
|
|
|
breeze_2.11-0.13.2.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
calcite-avatica-1.2.0-incubating.jar
|
|
|
|
calcite-core-1.2.0-incubating.jar
|
|
|
|
calcite-linq4j-1.2.0-incubating.jar
|
2017-09-01 14:21:21 -04:00
|
|
|
chill-java-0.8.4.jar
|
|
|
|
chill_2.11-0.8.4.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
commons-beanutils-1.7.0.jar
|
|
|
|
commons-beanutils-core-1.8.0.jar
|
|
|
|
commons-cli-1.2.jar
|
|
|
|
commons-codec-1.10.jar
|
|
|
|
commons-collections-3.2.2.jar
|
2017-12-06 19:15:25 -05:00
|
|
|
commons-compiler-3.0.8.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
commons-compress-1.4.1.jar
|
|
|
|
commons-configuration-1.6.jar
|
2016-08-30 12:15:31 -04:00
|
|
|
commons-crypto-1.0.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
commons-dbcp-1.4.jar
|
|
|
|
commons-digester-1.8.jar
|
|
|
|
commons-httpclient-3.1.jar
|
|
|
|
commons-io-2.4.jar
|
|
|
|
commons-lang-2.6.jar
|
2016-10-19 05:06:43 -04:00
|
|
|
commons-lang3-3.5.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
commons-logging-1.1.3.jar
|
|
|
|
commons-math3-3.4.1.jar
|
2018-02-27 09:18:41 -05:00
|
|
|
commons-net-3.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
commons-pool-1.5.4.jar
|
|
|
|
compress-lzf-1.0.3.jar
|
|
|
|
core-1.1.2.jar
|
2017-06-11 05:05:47 -04:00
|
|
|
curator-client-2.7.1.jar
|
|
|
|
curator-framework-2.7.1.jar
|
|
|
|
curator-recipes-2.7.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
datanucleus-api-jdo-3.2.6.jar
|
|
|
|
datanucleus-core-3.2.10.jar
|
|
|
|
datanucleus-rdbms-3.2.9.jar
|
2016-07-29 07:43:01 -04:00
|
|
|
derby-10.12.1.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
eigenbase-properties-1.1.5.jar
|
2017-07-10 18:21:03 -04:00
|
|
|
flatbuffers-1.2.0-3f79e055.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
generex-1.0.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
gson-2.2.4.jar
|
2016-04-04 19:52:21 -04:00
|
|
|
guava-14.0.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
guice-3.0.jar
|
|
|
|
guice-servlet-3.0.jar
|
2016-09-16 14:24:26 -04:00
|
|
|
hadoop-annotations-2.7.3.jar
|
|
|
|
hadoop-auth-2.7.3.jar
|
|
|
|
hadoop-client-2.7.3.jar
|
|
|
|
hadoop-common-2.7.3.jar
|
|
|
|
hadoop-hdfs-2.7.3.jar
|
|
|
|
hadoop-mapreduce-client-app-2.7.3.jar
|
|
|
|
hadoop-mapreduce-client-common-2.7.3.jar
|
|
|
|
hadoop-mapreduce-client-core-2.7.3.jar
|
|
|
|
hadoop-mapreduce-client-jobclient-2.7.3.jar
|
|
|
|
hadoop-mapreduce-client-shuffle-2.7.3.jar
|
|
|
|
hadoop-yarn-api-2.7.3.jar
|
|
|
|
hadoop-yarn-client-2.7.3.jar
|
|
|
|
hadoop-yarn-common-2.7.3.jar
|
|
|
|
hadoop-yarn-server-common-2.7.3.jar
|
|
|
|
hadoop-yarn-server-web-proxy-2.7.3.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
hk2-api-2.4.0-b34.jar
|
|
|
|
hk2-locator-2.4.0-b34.jar
|
|
|
|
hk2-utils-2.4.0-b34.jar
|
2017-12-21 06:43:56 -05:00
|
|
|
hppc-0.7.2.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
htrace-core-3.1.0-incubating.jar
|
2017-12-30 11:37:41 -05:00
|
|
|
httpclient-4.5.4.jar
|
|
|
|
httpcore-4.4.8.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
ivy-2.4.0.jar
|
2017-08-08 21:15:29 -04:00
|
|
|
jackson-annotations-2.6.7.jar
|
|
|
|
jackson-core-2.6.7.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jackson-core-asl-1.9.13.jar
|
2017-08-08 21:15:29 -04:00
|
|
|
jackson-databind-2.6.7.1.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
jackson-dataformat-yaml-2.6.7.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jackson-jaxrs-1.9.13.jar
|
|
|
|
jackson-mapper-asl-1.9.13.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
jackson-module-jaxb-annotations-2.6.7.jar
|
2017-09-28 03:22:48 -04:00
|
|
|
jackson-module-paranamer-2.7.9.jar
|
2017-08-08 21:15:29 -04:00
|
|
|
jackson-module-scala_2.11-2.6.7.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jackson-xc-1.9.13.jar
|
2017-12-06 19:15:25 -05:00
|
|
|
janino-3.0.8.jar
|
2017-12-02 08:37:02 -05:00
|
|
|
java-xmlbuilder-1.1.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
javassist-3.18.1-GA.jar
|
|
|
|
javax.annotation-api-1.2.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
javax.inject-1.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
javax.inject-2.4.0-b34.jar
|
2016-05-12 15:07:44 -04:00
|
|
|
javax.servlet-api-3.1.0.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
javax.ws.rs-api-2.0.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
javolution-5.5.1.jar
|
|
|
|
jaxb-api-2.2.2.jar
|
2016-02-17 22:03:29 -05:00
|
|
|
jcl-over-slf4j-1.7.16.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jdo-api-3.0.1.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
jersey-client-2.22.2.jar
|
|
|
|
jersey-common-2.22.2.jar
|
|
|
|
jersey-container-servlet-2.22.2.jar
|
|
|
|
jersey-container-servlet-core-2.22.2.jar
|
|
|
|
jersey-guava-2.22.2.jar
|
|
|
|
jersey-media-jaxb-2.22.2.jar
|
|
|
|
jersey-server-2.22.2.jar
|
2017-12-02 08:37:02 -05:00
|
|
|
jets3t-0.9.4.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jetty-6.1.26.jar
|
|
|
|
jetty-util-6.1.26.jar
|
2016-08-03 20:07:10 -04:00
|
|
|
jline-2.12.1.jar
|
2016-04-21 06:32:27 -04:00
|
|
|
joda-time-2.9.3.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jodd-core-3.5.2.jar
|
|
|
|
jpam-1.1.jar
|
2018-01-13 10:40:00 -05:00
|
|
|
json4s-ast_2.11-3.5.3.jar
|
|
|
|
json4s-core_2.11-3.5.3.jar
|
|
|
|
json4s-jackson_2.11-3.5.3.jar
|
|
|
|
json4s-scalap_2.11-3.5.3.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
jsp-api-2.1.jar
|
|
|
|
jsr305-1.3.9.jar
|
|
|
|
jta-1.1.jar
|
|
|
|
jtransforms-2.4.0.jar
|
2016-02-17 22:03:29 -05:00
|
|
|
jul-to-slf4j-1.7.16.jar
|
2016-04-08 19:35:30 -04:00
|
|
|
kryo-shaded-3.0.3.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
kubernetes-client-3.0.0.jar
|
|
|
|
kubernetes-model-2.0.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
leveldbjni-all-1.8.jar
|
2017-01-10 15:40:21 -05:00
|
|
|
libfb303-0.9.3.jar
|
|
|
|
libthrift-0.9.3.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
log4j-1.2.17.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
logging-interceptor-3.8.1.jar
|
2017-08-09 11:31:52 -04:00
|
|
|
lz4-java-1.4.0.jar
|
2017-04-25 13:10:41 -04:00
|
|
|
machinist_2.11-0.6.1.jar
|
|
|
|
macro-compat_2.11-1.1.1.jar
|
[SPARK-18935][MESOS] Fix dynamic reservations on mesos
## What changes were proposed in this pull request?
- Solves the issue described in the ticket by preserving reservation and allocation info in all cases (port handling included).
- upgrades to 1.4
- Adds extra debug level logging to make debugging easier in the future, for example we add reservation info when applicable.
```
17/09/29 14:53:07 DEBUG MesosCoarseGrainedSchedulerBackend: Accepting offer: f20de49b-dee3-45dd-a3c1-73418b7de891-O32 with attributes: Map() allocation info: role: "spark-prive"
reservation info: name: "ports"
type: RANGES
ranges {
range {
begin: 31000
end: 32000
}
}
role: "spark-prive"
reservation {
principal: "test"
}
allocation_info {
role: "spark-prive"
}
```
- Some style cleanup.
## How was this patch tested?
Manually by running the example in the ticket with and without a principal. Specifically I tested it on a dc/os 1.10 cluster with 7 nodes and played with reservations. From the master node in order to reserve resources I executed:
```for i in 0 1 2 3 4 5 6
do
curl -i \
-d slaveId=90ec65ea-1f7b-479f-a824-35d2527d6d26-S$i \
-d resources='[
{
"name": "cpus",
"type": "SCALAR",
"scalar": { "value": 2 },
"role": "spark-role",
"reservation": {
"principal": ""
}
},
{
"name": "mem",
"type": "SCALAR",
"scalar": { "value": 8026 },
"role": "spark-role",
"reservation": {
"principal": ""
}
}
]' \
-X POST http://master.mesos:5050/master/reserve
done
```
Nodes had 4 cpus (m3.xlarge instances) and I reserved either 2 or 4 cpus (all for a role).
I verified it launches tasks on nodes with reserved resources under `spark-role` role only if
a) there are remaining resources for (*) default role and the spark driver has no role assigned to it.
b) the spark driver has a role assigned to it and it is the same role used in reservations.
I also tested this locally on my machine.
Author: Stavros Kontopoulos <st.kontopoulos@gmail.com>
Closes #19390 from skonto/fix_dynamic_reservation.
2017-11-29 17:15:35 -05:00
|
|
|
mesos-1.4.0-shaded-protobuf.jar
|
2017-09-18 22:05:59 -04:00
|
|
|
metrics-core-3.1.5.jar
|
|
|
|
metrics-graphite-3.1.5.jar
|
|
|
|
metrics-json-3.1.5.jar
|
|
|
|
metrics-jvm-3.1.5.jar
|
2016-04-08 19:35:30 -04:00
|
|
|
minlog-1.3.0.jar
|
2016-12-03 04:53:47 -05:00
|
|
|
netty-3.9.9.Final.jar
|
2017-12-21 06:43:56 -05:00
|
|
|
netty-all-4.1.17.Final.jar
|
2016-04-08 19:35:30 -04:00
|
|
|
objenesis-2.1.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
okhttp-3.8.1.jar
|
|
|
|
okio-1.13.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
opencsv-2.3.jar
|
2018-02-17 03:25:36 -05:00
|
|
|
orc-core-1.4.3-nohive.jar
|
|
|
|
orc-mapreduce-1.4.3-nohive.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
oro-2.0.8.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
osgi-resource-locator-1.0.1.jar
|
2017-09-28 03:22:48 -04:00
|
|
|
paranamer-2.8.jar
|
2018-05-09 00:27:32 -04:00
|
|
|
parquet-column-1.10.0.jar
|
|
|
|
parquet-common-1.10.0.jar
|
|
|
|
parquet-encoding-1.10.0.jar
|
|
|
|
parquet-format-2.4.0.jar
|
|
|
|
parquet-hadoop-1.10.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
parquet-hadoop-bundle-1.6.0.jar
|
2018-05-09 00:27:32 -04:00
|
|
|
parquet-jackson-1.10.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
protobuf-java-2.5.0.jar
|
2018-04-13 17:28:24 -04:00
|
|
|
py4j-0.10.7.jar
|
2016-10-11 02:29:52 -04:00
|
|
|
pyrolite-4.13.jar
|
2016-04-01 18:21:29 -04:00
|
|
|
scala-compiler-2.11.8.jar
|
|
|
|
scala-library-2.11.8.jar
|
2016-01-30 03:20:28 -05:00
|
|
|
scala-parser-combinators_2.11-1.0.4.jar
|
2016-04-01 18:21:29 -04:00
|
|
|
scala-reflect-2.11.8.jar
|
2017-09-01 14:21:21 -04:00
|
|
|
scala-xml_2.11-1.0.5.jar
|
2017-04-25 13:10:41 -04:00
|
|
|
shapeless_2.11-2.3.2.jar
|
2016-02-17 22:03:29 -05:00
|
|
|
slf4j-api-1.7.16.jar
|
|
|
|
slf4j-log4j12-1.7.16.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
snakeyaml-1.15.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
snappy-0.2.jar
|
2018-02-08 13:52:08 -05:00
|
|
|
snappy-java-1.1.7.1.jar
|
2017-04-25 13:10:41 -04:00
|
|
|
spire-macros_2.11-0.13.0.jar
|
|
|
|
spire_2.11-0.13.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
stax-api-1.0-2.jar
|
|
|
|
stax-api-1.0.1.jar
|
|
|
|
stream-2.7.0.jar
|
2016-03-31 12:25:09 -04:00
|
|
|
stringtemplate-3.2.1.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
super-csv-2.2.0.jar
|
2018-05-13 22:01:06 -04:00
|
|
|
univocity-parsers-2.6.3.jar
|
2016-05-05 05:51:03 -04:00
|
|
|
validation-api-1.1.0.Final.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
xbean-asm5-shaded-4.4.jar
|
|
|
|
xercesImpl-2.9.1.jar
|
|
|
|
xmlenc-0.52.jar
|
|
|
|
xz-1.0.jar
|
2018-01-14 00:34:28 -05:00
|
|
|
zjsonpatch-0.3.0.jar
|
2016-01-15 20:07:24 -05:00
|
|
|
zookeeper-3.4.6.jar
|
2017-11-01 09:54:08 -04:00
|
|
|
zstd-jni-1.3.2-2.jar
|