big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-5199 Add more debug trace for NFS READ and WRITE. Contributed by Brandon Li

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1523140 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Brandon Li 2013-09-13 23:14:52 +00:00
parent 1260d598d3
commit a56a4b6ef0
6 changed files with 39 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
View File

@ -39,6 +39,12 @@
public class Nfs3Utils { public class Nfs3Utils {
public final static String INODEID_PATH_PREFIX = "/.reserved/.inodes/"; public final static String INODEID_PATH_PREFIX = "/.reserved/.inodes/";
public final static String READ_RPC_START = "READ_RPC_CALL_START____";
public final static String READ_RPC_END = "READ_RPC_CALL_END______";
public final static String WRITE_RPC_START = "WRITE_RPC_CALL_START____";
public final static String WRITE_RPC_END = "WRITE_RPC_CALL_END______";
public static String getFileIdPath(FileHandle handle) { public static String getFileIdPath(FileHandle handle) {
return getFileIdPath(handle.getFileId()); return getFileIdPath(handle.getFileId());
} }
@ -102,7 +108,10 @@ public static WccData createWccData(final WccAttr preOpAttr,
/** /**
* Send a write response to the netty network socket channel * Send a write response to the netty network socket channel
*/ */
public static void writeChannel(Channel channel, XDR out) { public static void writeChannel(Channel channel, XDR out, int xid) {
if (RpcProgramNfs3.LOG.isDebugEnabled()) {
RpcProgramNfs3.LOG.debug(WRITE_RPC_END + xid);
}
ChannelBuffer outBuf = XDR.writeMessageTcp(out, true); ChannelBuffer outBuf = XDR.writeMessageTcp(out, true);
channel.write(outBuf); channel.write(outBuf);
} }

16
hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
View File

@ -291,7 +291,7 @@ public void receivedNewWrite(DFSClient dfsClient, WRITE3Request request,
WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr); WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
fileWcc, 0, request.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF); fileWcc, 0, request.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
} else { } else {
// Handle repeated write requests(same xid or not). // Handle repeated write requests(same xid or not).
// If already replied, send reply again. If not replied, drop the // If already replied, send reply again. If not replied, drop the
@ -313,7 +313,7 @@ public void receivedNewWrite(DFSClient dfsClient, WRITE3Request request,
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, request.getCount(), request.getStableHow(), fileWcc, request.getCount(), request.getStableHow(),
Nfs3Constant.WRITE_COMMIT_VERF); Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
} }
updateLastAccessTime(); updateLastAccessTime();
@ -367,7 +367,7 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
WccData fileWcc = new WccData(preOpAttr, postOpAttr); WccData fileWcc = new WccData(preOpAttr, postOpAttr);
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF); fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
writeCtx.setReplied(true); writeCtx.setReplied(true);
} }
@ -392,7 +392,7 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
WccData fileWcc = new WccData(preOpAttr, postOpAttr); WccData fileWcc = new WccData(preOpAttr, postOpAttr);
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF); fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
writeCtx.setReplied(true); writeCtx.setReplied(true);
} }
@ -418,7 +418,7 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
} }
updateLastAccessTime(); updateLastAccessTime();
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
} }
} }
@ -707,7 +707,7 @@ private void doSingleWrite(final WriteCtx writeCtx) {
WccData fileWcc = new WccData(preOpAttr, latestAttr); WccData fileWcc = new WccData(preOpAttr, latestAttr);
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF); fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
} }
} catch (IOException e) { } catch (IOException e) {
@ -715,7 +715,7 @@ private void doSingleWrite(final WriteCtx writeCtx) {
+ offset + " and length " + data.length, e); + offset + " and length " + data.length, e);
if (!writeCtx.getReplied()) { if (!writeCtx.getReplied()) {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO); WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
// Keep stream open. Either client retries or SteamMonitor closes it. // Keep stream open. Either client retries or SteamMonitor closes it.
} }
@ -753,7 +753,7 @@ private void cleanup() {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
fileWcc, 0, writeCtx.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF); fileWcc, 0, writeCtx.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(writeCtx.getChannel(), Nfs3Utils.writeChannel(writeCtx.getChannel(),
response.send(new XDR(), writeCtx.getXid())); response.send(new XDR(), writeCtx.getXid()), writeCtx.getXid());
} }
} }

13
hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
View File

@ -125,7 +125,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
public static final FsPermission umask = new FsPermission( public static final FsPermission umask = new FsPermission(
(short) DEFAULT_UMASK); (short) DEFAULT_UMASK);
private static final Log LOG = LogFactory.getLog(RpcProgramNfs3.class); static final Log LOG = LogFactory.getLog(RpcProgramNfs3.class);
private static final int MAX_READ_TRANSFER_SIZE = 64 * 1024; private static final int MAX_READ_TRANSFER_SIZE = 64 * 1024;
private static final int MAX_WRITE_TRANSFER_SIZE = 64 * 1024; private static final int MAX_WRITE_TRANSFER_SIZE = 64 * 1024;
private static final int MAX_READDIR_TRANSFER_SIZE = 64 * 1024; private static final int MAX_READDIR_TRANSFER_SIZE = 64 * 1024;
@ -1814,9 +1814,19 @@ public XDR handleInternal(RpcCall rpcCall, final XDR xdr, XDR out,
} else if (nfsproc3 == NFSPROC3.READLINK) { } else if (nfsproc3 == NFSPROC3.READLINK) {
response = readlink(xdr, securityHandler, client); response = readlink(xdr, securityHandler, client);
} else if (nfsproc3 == NFSPROC3.READ) { } else if (nfsproc3 == NFSPROC3.READ) {
if (LOG.isDebugEnabled()) {
LOG.debug(Nfs3Utils.READ_RPC_START + xid);
}
response = read(xdr, securityHandler, client); response = read(xdr, securityHandler, client);
if (LOG.isDebugEnabled() && (nfsproc3 == NFSPROC3.READ)) {
LOG.debug(Nfs3Utils.READ_RPC_END + xid);
}
} else if (nfsproc3 == NFSPROC3.WRITE) { } else if (nfsproc3 == NFSPROC3.WRITE) {
if (LOG.isDebugEnabled()) {
LOG.debug(Nfs3Utils.WRITE_RPC_START + xid);
}
response = write(xdr, channel, xid, securityHandler, client); response = write(xdr, channel, xid, securityHandler, client);
// Write end debug trace is in Nfs3Utils.writeChannel
} else if (nfsproc3 == NFSPROC3.CREATE) { } else if (nfsproc3 == NFSPROC3.CREATE) {
response = create(xdr, securityHandler, client); response = create(xdr, securityHandler, client);
} else if (nfsproc3 == NFSPROC3.MKDIR) { } else if (nfsproc3 == NFSPROC3.MKDIR) {
@ -1853,6 +1863,7 @@ public XDR handleInternal(RpcCall rpcCall, final XDR xdr, XDR out,
if (response != null) { if (response != null) {
out = response.send(out, xid); out = response.send(out, xid);
} }
return out; return out;
} }

8
hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
View File

@ -118,7 +118,7 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
byte[] data = request.getData().array(); byte[] data = request.getData().array();
if (data.length < count) { if (data.length < count) {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_INVAL); WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_INVAL);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
return; return;
} }
@ -155,7 +155,7 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
fileWcc, count, request.getStableHow(), fileWcc, count, request.getStableHow(),
Nfs3Constant.WRITE_COMMIT_VERF); Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
return; return;
} }
@ -182,10 +182,10 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK, WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, count, request.getStableHow(), fileWcc, count, request.getStableHow(),
Nfs3Constant.WRITE_COMMIT_VERF); Nfs3Constant.WRITE_COMMIT_VERF);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
} else { } else {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO); WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO);
Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid)); Nfs3Utils.writeChannel(channel, response.send(new XDR(), xid), xid);
} }
} }

6
hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
View File

@ -174,11 +174,11 @@ public static void main(String[] args) throws InterruptedException {
XDR writeReq; XDR writeReq;
writeReq = write(handle, 0x8000005c, 2000, 1000, data3); writeReq = write(handle, 0x8000005c, 2000, 1000, data3);
Nfs3Utils.writeChannel(channel, writeReq); Nfs3Utils.writeChannel(channel, writeReq, 1);
writeReq = write(handle, 0x8000005d, 1000, 1000, data2); writeReq = write(handle, 0x8000005d, 1000, 1000, data2);
Nfs3Utils.writeChannel(channel, writeReq); Nfs3Utils.writeChannel(channel, writeReq, 2);
writeReq = write(handle, 0x8000005e, 0, 1000, data1); writeReq = write(handle, 0x8000005e, 0, 1000, data1);
Nfs3Utils.writeChannel(channel, writeReq); Nfs3Utils.writeChannel(channel, writeReq, 3);
// TODO: convert to Junit test, and validate result automatically // TODO: convert to Junit test, and validate result automatically
} }

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -338,6 +338,8 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5067 Support symlink operations in NFS gateway. (brandonli) HDFS-5067 Support symlink operations in NFS gateway. (brandonli)
HDFS-5199 Add more debug trace for NFS READ and WRITE. (brandonli)
IMPROVEMENTS IMPROVEMENTS
HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may