ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-19042] spark executor can't download the jars when uber jar's http url contains any query strings

Browse source 
If the uber jars' https contains any query strings, the Executor.updateDependencies method can't can't download the jars correctly. This is because  the "localName = name.split("/").last" won't get the expected jar's url. The bug fix is the same as [SPARK-17855]

Author: xiaojian.fxj <xiaojian.fxj@alibaba-inc.com>

Closes #16509 from hustfxj/bug.
This commit is contained in:
xiaojian.fxj 2017-01-15 11:12:59 +00:00 committed by Sean Owen
parent 9112f31bb8
commit c9d612f82c
No known key found for this signature in database
GPG key ID: BEB3956D6717BDDC
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
core/src/main/scala/org/apache/spark/executor/Executor.scala
View file

@ -19,7 +19,7 @@ package org.apache.spark.executor
import java.io.{File, NotSerializableException} import java.io.{File, NotSerializableException}
import java.lang.management.ManagementFactory import java.lang.management.ManagementFactory
import java.net.URL import java.net.{URI, URL}
import java.nio.ByteBuffer import java.nio.ByteBuffer
import java.util.Properties import java.util.Properties
import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
@ -640,7 +640,7 @@ private[spark] class Executor(
currentFiles(name) = timestamp currentFiles(name) = timestamp
} }
for ((name, timestamp) <- newJars) { for ((name, timestamp) <- newJars) {
val localName = name.split("/").last val localName = new URI(name).getPath.split("/").last
val currentTimeStamp = currentJars.get(name) val currentTimeStamp = currentJars.get(name)
.orElse(currentJars.get(localName)) .orElse(currentJars.get(localName))
.getOrElse(-1L) .getOrElse(-1L)