ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-19857][YARN] Correctly calculate next credential update time.

Browse source 
Add parentheses so that both lines form a single statement; also add
a log message so that the issue becomes more explicit if it shows up
again.

Tested manually with integration test that exercises the feature.

Author: Marcelo Vanzin <vanzin@cloudera.com>

Closes #17198 from vanzin/SPARK-19857.
This commit is contained in:
Marcelo Vanzin 2017-03-07 16:21:18 -08:00
parent 2e30c0b9bc
commit 8e41c2eed8
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala
View file

@ -60,7 +60,7 @@ private[spark] class CredentialUpdater(
if (remainingTime <= 0) { if (remainingTime <= 0) {
credentialUpdater.schedule(credentialUpdaterRunnable, 1, TimeUnit.MINUTES) credentialUpdater.schedule(credentialUpdaterRunnable, 1, TimeUnit.MINUTES)
} else { } else {
logInfo(s"Scheduling credentials refresh from HDFS in $remainingTime millis.") logInfo(s"Scheduling credentials refresh from HDFS in $remainingTime ms.")
credentialUpdater.schedule(credentialUpdaterRunnable, remainingTime, TimeUnit.MILLISECONDS) credentialUpdater.schedule(credentialUpdaterRunnable, remainingTime, TimeUnit.MILLISECONDS)
} }
} }
@ -81,8 +81,8 @@ private[spark] class CredentialUpdater(
UserGroupInformation.getCurrentUser.addCredentials(newCredentials) UserGroupInformation.getCurrentUser.addCredentials(newCredentials)
logInfo("Credentials updated from credentials file.") logInfo("Credentials updated from credentials file.")
val remainingTime = getTimeOfNextUpdateFromFileName(credentialsStatus.getPath) val remainingTime = (getTimeOfNextUpdateFromFileName(credentialsStatus.getPath)
- System.currentTimeMillis() - System.currentTimeMillis())
if (remainingTime <= 0) TimeUnit.MINUTES.toMillis(1) else remainingTime if (remainingTime <= 0) TimeUnit.MINUTES.toMillis(1) else remainingTime
} else { } else {
// If current credential file is older than expected, sleep 1 hour and check again. // If current credential file is older than expected, sleep 1 hour and check again.
@ -100,6 +100,7 @@ private[spark] class CredentialUpdater(
TimeUnit.HOURS.toMillis(1) TimeUnit.HOURS.toMillis(1)
} }
logInfo(s"Scheduling credentials refresh from HDFS in $timeToNextUpdate ms.")
credentialUpdater.schedule( credentialUpdater.schedule(
credentialUpdaterRunnable, timeToNextUpdate, TimeUnit.MILLISECONDS) credentialUpdaterRunnable, timeToNextUpdate, TimeUnit.MILLISECONDS)
} }