HDFS-4594. WebHDFS open sets Content-Length header to what is specified by length parameter rather than how much data is actually returned. Contributed by Chris Nauroth.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1515989 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -372,6 +372,9 @@ Release 2.1.1-beta - UNRELEASED
     HDFS-5106. TestDatanodeBlockScanner fails on Windows due to incorrect path
     format. (Chuan Liu via cnauroth)
 
+    HDFS-4594. WebHDFS open sets Content-Length header to what is specified by
+    length parameter rather than how much data is actually returned. (cnauroth)
+
 Release 2.1.0-beta - 2013-08-22
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java

@@ -410,8 +410,9 @@ private Response get(
         throw ioe;
       }
       
-      final long n = length.getValue() != null? length.getValue()
+      final long n = length.getValue() != null ?
-        : in.getVisibleLength() - offset.getValue();
+        Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
+        in.getVisibleLength() - offset.getValue();
       return Response.ok(new OpenEntity(in, n, dfsclient)).type(
           MediaType.APPLICATION_OCTET_STREAM).build();
     }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java

@@ -21,6 +21,7 @@
 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -45,8 +46,11 @@
 import org.apache.hadoop.hdfs.web.resources.DoAsParam;
 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.LengthParam;
 import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
+import org.apache.hadoop.hdfs.web.resources.OffsetParam;
 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.Assert;
@@ -288,6 +292,104 @@ public void testRootDir() throws IOException {
     }
   }
 
+  /**
+   * Test get with length parameter greater than actual file length.
+   */
+  public void testLengthParamLongerThanFile() throws IOException {
+    WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs;
+    Path dir = new Path("/test");
+    assertTrue(webhdfs.mkdirs(dir));
+
+    // Create a file with some content.
+    Path testFile = new Path("/test/testLengthParamLongerThanFile");
+    String content = "testLengthParamLongerThanFile";
+    FSDataOutputStream testFileOut = webhdfs.create(testFile);
+    try {
+      testFileOut.write(content.getBytes("US-ASCII"));
+    } finally {
+      IOUtils.closeStream(testFileOut);
+    }
+
+    // Open the file, but request length longer than actual file length by 1.
+    HttpOpParam.Op op = GetOpParam.Op.OPEN;
+    URL url = webhdfs.toUrl(op, testFile, new LengthParam(Long.valueOf(
+      content.length() + 1)));
+    HttpURLConnection conn = null;
+    InputStream is = null;
+    try {
+      conn = (HttpURLConnection)url.openConnection();
+      conn.setRequestMethod(op.getType().toString());
+      conn.setDoOutput(op.getDoOutput());
+      conn.setInstanceFollowRedirects(true);
+
+      // Expect OK response and Content-Length header equal to actual length.
+      assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode());
+      assertEquals(String.valueOf(content.length()), conn.getHeaderField(
+        "Content-Length"));
+
+      // Check content matches.
+      byte[] respBody = new byte[content.length()];
+      is = conn.getInputStream();
+      IOUtils.readFully(is, respBody, 0, content.length());
+      assertEquals(content, new String(respBody, "US-ASCII"));
+    } finally {
+      IOUtils.closeStream(is);
+      if (conn != null) {
+        conn.disconnect();
+      }
+    }
+  }
+
+  /**
+   * Test get with offset and length parameters that combine to request a length
+   * greater than actual file length.
+   */
+  public void testOffsetPlusLengthParamsLongerThanFile() throws IOException {
+    WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs;
+    Path dir = new Path("/test");
+    assertTrue(webhdfs.mkdirs(dir));
+
+    // Create a file with some content.
+    Path testFile = new Path("/test/testOffsetPlusLengthParamsLongerThanFile");
+    String content = "testOffsetPlusLengthParamsLongerThanFile";
+    FSDataOutputStream testFileOut = webhdfs.create(testFile);
+    try {
+      testFileOut.write(content.getBytes("US-ASCII"));
+    } finally {
+      IOUtils.closeStream(testFileOut);
+    }
+
+    // Open the file, but request offset starting at 1 and length equal to file
+    // length.  Considering the offset, this is longer than the actual content.
+    HttpOpParam.Op op = GetOpParam.Op.OPEN;
+    URL url = webhdfs.toUrl(op, testFile, new LengthParam(Long.valueOf(
+      content.length())), new OffsetParam(1L));
+    HttpURLConnection conn = null;
+    InputStream is = null;
+    try {
+      conn = (HttpURLConnection)url.openConnection();
+      conn.setRequestMethod(op.getType().toString());
+      conn.setDoOutput(op.getDoOutput());
+      conn.setInstanceFollowRedirects(true);
+
+      // Expect OK response and Content-Length header equal to actual length.
+      assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode());
+      assertEquals(String.valueOf(content.length() - 1), conn.getHeaderField(
+        "Content-Length"));
+
+      // Check content matches.
+      byte[] respBody = new byte[content.length() - 1];
+      is = conn.getInputStream();
+      IOUtils.readFully(is, respBody, 0, content.length() - 1);
+      assertEquals(content.substring(1), new String(respBody, "US-ASCII"));
+    } finally {
+      IOUtils.closeStream(is);
+      if (conn != null) {
+        conn.disconnect();
+      }
+    }
+  }
+
   public void testResponseCode() throws IOException {
     final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs;
     final Path root = new Path("/");