[SPARK-36332][SHUFFLE] Cleanup RemoteBlockPushResolver log messages

### What changes were proposed in this pull request?
Cleanup `RemoteBlockPushResolver` log messages by using `AppShufflePartitionInfo#toString()` to avoid duplications. Currently this is based off of https://github.com/apache/spark/pull/33034 will remove those changes once it is merged and remove the WIP at that time.

### Why are the changes needed?
Minor cleanup to make code more readable.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
No tests, just changing log messages

Closes #33561 from venkata91/SPARK-36332.

Authored-by: Venkata krishnan Sowrirajan <vsowrirajan@linkedin.com>
Signed-off-by: yi.wu <yi.wu@databricks.com>
(cherry picked from commit ab897109a3)
Signed-off-by: yi.wu <yi.wu@databricks.com>
This commit is contained in:
Venkata krishnan Sowrirajan 2021-08-10 09:53:53 +08:00 committed by yi.wu
parent 10f7f6e62b
commit 38dd42a50c

View file

@ -684,9 +684,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
private void writeBuf(ByteBuffer buf) throws IOException {
while (buf.hasRemaining()) {
long updatedPos = partitionInfo.getDataFilePos() + length;
logger.debug("{} shuffleId {} shuffleMergeId {} reduceId {} current pos"
+ " {} updated pos {}", partitionInfo.appId, partitionInfo.shuffleId,
partitionInfo.shuffleMergeId, partitionInfo.reduceId,
logger.debug("{} current pos {} updated pos {}", partitionInfo,
partitionInfo.getDataFilePos(), updatedPos);
length += partitionInfo.dataChannel.write(buf, updatedPos);
}
@ -801,9 +799,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
return;
}
abortIfNecessary();
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} onData writable",
partitionInfo.appId, partitionInfo.shuffleId, partitionInfo.shuffleMergeId,
partitionInfo.reduceId);
logger.trace("{} onData writable", partitionInfo);
if (partitionInfo.getCurrentMapIndex() < 0) {
partitionInfo.setCurrentMapIndex(mapIndex);
}
@ -823,9 +819,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
throw ioe;
}
} else {
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} onData deferred",
partitionInfo.appId, partitionInfo.shuffleId, partitionInfo.shuffleMergeId,
partitionInfo.reduceId);
logger.trace("{} onData deferred", partitionInfo);
// If we cannot write to disk, we buffer the current block chunk in memory so it could
// potentially be written to disk later. We take our best effort without guarantee
// that the block will be written to disk. If the block data is divided into multiple
@ -858,9 +852,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
@Override
public void onComplete(String streamId) throws IOException {
synchronized (partitionInfo) {
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} onComplete invoked",
partitionInfo.appId, partitionInfo.shuffleId, partitionInfo.shuffleMergeId,
partitionInfo.reduceId);
logger.trace("{} onComplete invoked", partitionInfo);
// Initially when this request got to the server, the shuffle merge finalize request
// was not received yet or this was the latest stage attempt (or latest shuffleMergeId)
// generating shuffle output for the shuffle ID. By the time we finish reading this
@ -942,9 +934,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
synchronized (partitionInfo) {
if (!isStaleOrTooLate(appShuffleInfo.shuffles.get(partitionInfo.shuffleId),
partitionInfo.shuffleMergeId, partitionInfo.reduceId)) {
logger.debug("{} shuffleId {} shuffleMergeId {} reduceId {}"
+ " encountered failure", partitionInfo.appId, partitionInfo.shuffleId,
partitionInfo.shuffleMergeId, partitionInfo.reduceId);
logger.debug("{} encountered failure", partitionInfo);
partitionInfo.setCurrentMapIndex(-1);
}
}
@ -1038,9 +1028,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
}
public void setDataFilePos(long dataFilePos) {
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} current pos {}"
+ " update pos {}", appId, shuffleId, shuffleMergeId, reduceId, this.dataFilePos,
dataFilePos);
logger.trace("{} current pos {} update pos {}", this, this.dataFilePos, dataFilePos);
this.dataFilePos = dataFilePos;
}
@ -1049,9 +1037,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
}
void setCurrentMapIndex(int mapIndex) {
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} updated mapIndex {}"
+ " current mapIndex {}", appId, shuffleId, shuffleMergeId, reduceId,
currentMapIndex, mapIndex);
logger.trace("{} mapIndex {} current mapIndex {}", this, currentMapIndex, mapIndex);
this.currentMapIndex = mapIndex;
}
@ -1060,8 +1046,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
}
void blockMerged(int mapIndex) {
logger.debug("{} shuffleId {} shuffleMergeId {} reduceId {} updated merging mapIndex {}",
appId, shuffleId, shuffleMergeId, reduceId, mapIndex);
logger.debug("{} updated merging mapIndex {}", this, mapIndex);
mapTracker.add(mapIndex);
chunkTracker.add(mapIndex);
lastMergedMapIndex = mapIndex;
@ -1079,9 +1064,8 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
*/
void updateChunkInfo(long chunkOffset, int mapIndex) throws IOException {
try {
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} index current {}"
+ " updated {}", appId, shuffleId, shuffleMergeId, reduceId,
this.lastChunkOffset, chunkOffset);
logger.trace("{} index current {} updated {}", this, this.lastChunkOffset,
chunkOffset);
if (indexMetaUpdateFailed) {
indexFile.getChannel().position(indexFile.getPos());
}
@ -1109,8 +1093,7 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
return;
}
chunkTracker.add(mapIndex);
logger.trace("{} shuffleId {} shuffleMergeId {} reduceId {} mapIndex {}"
+ " write chunk to meta file", appId, shuffleId, shuffleMergeId, reduceId, mapIndex);
logger.trace("{} mapIndex {} write chunk to meta file", this, mapIndex);
if (indexMetaUpdateFailed) {
metaFile.getChannel().position(metaFile.getPos());
}
@ -1175,6 +1158,12 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
}
}
@Override
public String toString() {
return String.format("Application %s shuffleId %s shuffleMergeId %s reduceId %s",
appId, shuffleId, shuffleMergeId, reduceId);
}
@Override
protected void finalize() throws Throwable {
closeAllFilesAndDeleteIfNeeded(false);