big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline. Contributed by Eddy Xu.

Browse Source
This commit is contained in:
Andrew Wang 2015-06-11 10:12:31 -07:00
parent 95c73d49b1
commit b258b344bb
6 changed files with 40 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8549. Abort the balancer if an upgrade is in progress. (wang)
HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to
ReplicaInPipeline. (Eddy Xu via wang)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
View File

@ -23,7 +23,6 @@
import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileOutputStream;
import java.io.IOException;
@ -839,15 +838,8 @@ void receiveBlock(
// In case this datanode is shutting down for quick restart,
// send a special ack upstream.
if (datanode.isRestarting() && isClient && !isTransfer) {
File blockFile = ((ReplicaInPipeline)replicaInfo).getBlockFile();
File restartMeta = new File(blockFile.getParent() +
File.pathSeparator + "." + blockFile.getName() + ".restart");
if (restartMeta.exists() && !restartMeta.delete()) {
LOG.warn("Failed to delete restart meta file: " +
restartMeta.getPath());
}
try (Writer out = new OutputStreamWriter(
new FileOutputStream(restartMeta), "UTF-8")) {
replicaInfo.createRestartMetaStream(), "UTF-8")) {
// write out the current time.
out.write(Long.toString(Time.now() + restartBudget));
out.flush();

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
View File

@ -20,6 +20,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import org.apache.hadoop.hdfs.protocol.Block;
@ -287,7 +288,19 @@ public ReplicaOutputStreams createStreams(boolean isCreate,
throw e;
}
}
@Override
public OutputStream createRestartMetaStream() throws IOException {
File blockFile = getBlockFile();
File restartMeta = new File(blockFile.getParent() +
File.pathSeparator + "." + blockFile.getName() + ".restart");
if (restartMeta.exists() && !restartMeta.delete()) {
DataNode.LOG.warn("Failed to delete restart meta file: " +
restartMeta.getPath());
}
return new FileOutputStream(restartMeta);
}
@Override
public String toString() {
return super.toString()

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.util.DataChecksum;
@ -73,4 +74,13 @@ public interface ReplicaInPipelineInterface extends Replica {
*/
public ReplicaOutputStreams createStreams(boolean isCreate,
DataChecksum requestedChecksum) throws IOException;
/**
* Create an output stream to write restart metadata in case of datanode
* shutting down for quick restart.
*
* @return output stream for writing.
* @throws IOException if any error occurs
*/
public OutputStream createRestartMetaStream() throws IOException;
}

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
View File

@ -262,6 +262,11 @@ synchronized public ReplicaOutputStreams createStreams(boolean isCreate,
}
}
@Override
public OutputStream createRestartMetaStream() throws IOException {
return new SimulatedOutputStream();
}
@Override
synchronized public long getBlockId() {
return theBlock.getBlockId();

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
View File

@ -19,6 +19,8 @@
package org.apache.hadoop.hdfs.server.datanode.extdataset;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
@ -59,6 +61,11 @@ public ReplicaOutputStreams createStreams(boolean isCreate,
return new ReplicaOutputStreams(null, null, requestedChecksum, false);
}
@Override
public OutputStream createRestartMetaStream() throws IOException {
return null;
}
@Override
public long getBlockId() {
return 0;