big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15265. HttpFS: validate content-type in HttpFSUtils. Contributed by hemanthboyina.

Browse Source 
(cherry picked from commit 31b2f687ef)
This commit is contained in:
Inigo Goiri 2020-04-29 13:41:36 -07:00 committed by Wei-Chiu Chuang
parent 5187bd37ae
commit 41fad5056c
2 changed files with 43 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java
View File

@ -33,6 +33,8 @@
import java.util.List;
import java.util.Map;
import javax.ws.rs.core.MediaType;
/**
* Utility methods used by HttpFS classes.
*/
@ -127,8 +129,17 @@ static URL createURL(Path path, Map<String, String> params, Map<String,
* @throws IOException thrown if the <code>InputStream</code> could not be
* JSON parsed.
*/
static Object jsonParse(HttpURLConnection conn) throws IOException {
public static Object jsonParse(HttpURLConnection conn) throws IOException {
try {
String contentType = conn.getContentType();
if (contentType != null) {
final MediaType parsed = MediaType.valueOf(contentType);
if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
throw new IOException("Content-Type \"" + contentType
+ "\" is incompatible with \"" + MediaType.APPLICATION_JSON
+ "\" (parsed=\"" + parsed + "\")");
}
}
JSONParser parser = new JSONParser();
return parser.parse(
new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8));

31
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
View File

@ -71,6 +71,8 @@
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrCodec;
import org.apache.hadoop.fs.http.client.HttpFSUtils;
import org.apache.hadoop.fs.http.client.HttpFSFileSystem.Operation;
import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam;
import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.NoRedirectParam;
import org.apache.hadoop.fs.permission.AclEntry;
@ -89,6 +91,7 @@
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.HFSTestCase;
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.test.TestDir;
import org.apache.hadoop.test.TestDirHelper;
import org.apache.hadoop.test.TestHdfs;
@ -1917,4 +1920,32 @@ public void testNoRedirectWithData() throws Exception {
Assert.assertEquals(TestJettyHelper.getJettyURL() + "/webhdfs/v1" + path,
location);
}
@Test
@TestDir
@TestJetty
@TestHdfs
public void testContentType() throws Exception {
createHttpFSServer(false, false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
Path dir = new Path("/tmp");
Path file = new Path(dir, "foo");
fs.mkdirs(dir);
fs.create(file);
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format(
"/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2", user));
// test jsonParse with non-json type.
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod(Operation.OPEN.getMethod());
conn.connect();
LambdaTestUtils.intercept(IOException.class,
"Content-Type \"text/html;charset=iso-8859-1\" "
+ "is incompatible with \"application/json\"",
() -> HttpFSUtils.jsonParse(conn));
conn.disconnect();
}
}