diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java index 99924e5c16..8de4bb25f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java @@ -55,9 +55,13 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpContent chunk) throws IOException { chunk.content().readBytes(out, chunk.content().readableBytes()); if (chunk instanceof LastHttpContent) { - response.headers().set(CONNECTION, CLOSE); - ctx.write(response).addListener(ChannelFutureListener.CLOSE); - releaseDfsResources(); + try { + releaseDfsResourcesAndThrow(); + response.headers().set(CONNECTION, CLOSE); + ctx.write(response).addListener(ChannelFutureListener.CLOSE); + } catch (Exception cause) { + exceptionCaught(ctx, cause); + } } } @@ -71,7 +75,10 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { releaseDfsResources(); DefaultHttpResponse resp = ExceptionHandler.exceptionCaught(cause); resp.headers().set(CONNECTION, CLOSE); - ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); + ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE); + if (LOG != null && LOG.isDebugEnabled()) { + LOG.debug("Exception in channel handler ", cause); + } } private void releaseDfsResources() { @@ -79,4 +86,8 @@ private void releaseDfsResources() { IOUtils.cleanup(LOG, client); } + private void releaseDfsResourcesAndThrow() throws Exception { + out.close(); + client.close(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index cf02a8de84..51dca41014 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -1980,13 +1980,20 @@ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) { */ public void shutdownDataNodes() { for (int i = dataNodes.size()-1; i >= 0; i--) { - LOG.info("Shutting down DataNode " + i); - DataNode dn = dataNodes.remove(i).datanode; - dn.shutdown(); - numDataNodes--; + shutdownDataNode(i); } } + /** + * Shutdown the datanode at a given index. + */ + public void shutdownDataNode(int dnIndex) { + LOG.info("Shutting down DataNode " + dnIndex); + DataNode dn = dataNodes.remove(dnIndex).datanode; + dn.shutdown(); + numDataNodes--; + } + /** * Shutdown all the namenodes. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 259353c6aa..d4495dc422 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -1181,4 +1181,43 @@ public void testSetStoragePolicyWhenPolicyDisabled() throws Exception { cluster.shutdown(); } } + + @Test + public void testWebHdfsAppend() throws Exception { + MiniDFSCluster cluster = null; + final Configuration conf = WebHdfsTestUtil.createConf(); + final int dnNumber = 3; + try { + + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNumber).build(); + + final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem( + conf, WebHdfsConstants.WEBHDFS_SCHEME); + + final DistributedFileSystem fs = cluster.getFileSystem(); + + final Path appendFile = new Path("/testAppend.txt"); + final String content = "hello world"; + DFSTestUtil.writeFile(fs, appendFile, content); + + for (int index = 0; index < dnNumber - 1; index++){ + cluster.shutdownDataNode(index); + } + cluster.restartNameNodes(); + cluster.waitActive(); + + try { + DFSTestUtil.appendFile(webFS, appendFile, content); + fail("Should fail to append file since " + + "datanode number is 1 and replication is 3"); + } catch (IOException ignored) { + String resultContent = DFSTestUtil.readFile(fs, appendFile); + assertTrue(resultContent.equals(content)); + } + } finally { + if (cluster != null) { + cluster.shutdown(true); + } + } + } }