big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15275. HttpFS: Response of Create was not correct with noredirect and data are true. Contributed by hemanthboyina.

Browse Source 
(cherry picked from commit 1fdfaebd98)
This commit is contained in:
Takanobu Asanuma 2020-04-20 23:09:33 +09:00 committed by Wei-Chiu Chuang
parent 813aafcee2
commit 9a0c5d0ca2
2 changed files with 65 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
View File

@ -629,23 +629,21 @@ public Response post(InputStream is,
switch (op.value()) { switch (op.value()) {
case APPEND: { case APPEND: {
Boolean hasData = params.get(DataParam.NAME, DataParam.class); Boolean hasData = params.get(DataParam.NAME, DataParam.class);
if (!hasData) { URI redirectURL = createUploadRedirectionURL(uriInfo,
URI redirectURL = createUploadRedirectionURL( HttpFSFileSystem.Operation.APPEND);
uriInfo, HttpFSFileSystem.Operation.APPEND); Boolean noRedirect =
Boolean noRedirect = params.get( params.get(NoRedirectParam.NAME, NoRedirectParam.class);
NoRedirectParam.NAME, NoRedirectParam.class); if (noRedirect) {
if (noRedirect) {
final String js = JsonUtil.toJsonString("Location", redirectURL); final String js = JsonUtil.toJsonString("Location", redirectURL);
response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
} else { } else if (hasData) {
response = Response.temporaryRedirect(redirectURL).build();
}
} else {
FSOperations.FSAppend command = FSOperations.FSAppend command =
new FSOperations.FSAppend(is, path); new FSOperations.FSAppend(is, path);
fsExecute(user, command); fsExecute(user, command);
AUDIT_LOG.info("[{}]", path); AUDIT_LOG.info("[{}]", path);
response = Response.ok().type(MediaType.APPLICATION_JSON).build(); response = Response.ok().type(MediaType.APPLICATION_JSON).build();
} else {
response = Response.temporaryRedirect(redirectURL).build();
} }
break; break;
} }
@ -703,7 +701,8 @@ public Response post(InputStream is,
protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum<?> uploadOperation) { protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum<?> uploadOperation) {
UriBuilder uriBuilder = uriInfo.getRequestUriBuilder(); UriBuilder uriBuilder = uriInfo.getRequestUriBuilder();
uriBuilder = uriBuilder.replaceQueryParam(OperationParam.NAME, uploadOperation). uriBuilder = uriBuilder.replaceQueryParam(OperationParam.NAME, uploadOperation).
queryParam(DataParam.NAME, Boolean.TRUE); queryParam(DataParam.NAME, Boolean.TRUE)
.replaceQueryParam(NoRedirectParam.NAME, (Object[]) null);
return uriBuilder.build(null); return uriBuilder.build(null);
} }
@ -771,18 +770,14 @@ public Response put(InputStream is,
switch (op.value()) { switch (op.value()) {
case CREATE: { case CREATE: {
Boolean hasData = params.get(DataParam.NAME, DataParam.class); Boolean hasData = params.get(DataParam.NAME, DataParam.class);
if (!hasData) { URI redirectURL = createUploadRedirectionURL(uriInfo,
URI redirectURL = createUploadRedirectionURL( HttpFSFileSystem.Operation.CREATE);
uriInfo, HttpFSFileSystem.Operation.CREATE); Boolean noRedirect =
Boolean noRedirect = params.get( params.get(NoRedirectParam.NAME, NoRedirectParam.class);
NoRedirectParam.NAME, NoRedirectParam.class); if (noRedirect) {
if (noRedirect) {
final String js = JsonUtil.toJsonString("Location", redirectURL); final String js = JsonUtil.toJsonString("Location", redirectURL);
response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
} else { } else if (hasData) {
response = Response.temporaryRedirect(redirectURL).build();
}
} else {
Short permission = params.get(PermissionParam.NAME, Short permission = params.get(PermissionParam.NAME,
PermissionParam.class); PermissionParam.class);
Short unmaskedPermission = params.get(UnmaskedPermissionParam.NAME, Short unmaskedPermission = params.get(UnmaskedPermissionParam.NAME,
@ -806,6 +801,8 @@ public Response put(InputStream is,
"Location", uriInfo.getAbsolutePath()); "Location", uriInfo.getAbsolutePath());
response = Response.created(uriInfo.getAbsolutePath()) response = Response.created(uriInfo.getAbsolutePath())
.type(MediaType.APPLICATION_JSON).entity(js).build(); .type(MediaType.APPLICATION_JSON).entity(js).build();
} else {
response = Response.temporaryRedirect(redirectURL).build();
} }
break; break;
} }

48
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
View File

@ -1602,7 +1602,7 @@ public void testNoRedirect() throws Exception {
new InputStreamReader(conn.getInputStream())); new InputStreamReader(conn.getInputStream()));
String location = (String)json.get("Location"); String location = (String)json.get("Location");
Assert.assertTrue(location.contains(DataParam.NAME)); Assert.assertTrue(location.contains(DataParam.NAME));
Assert.assertTrue(location.contains(NoRedirectParam.NAME)); Assert.assertFalse(location.contains(NoRedirectParam.NAME));
Assert.assertTrue(location.contains("CREATE")); Assert.assertTrue(location.contains("CREATE"));
Assert.assertTrue("Wrong location: " + location, Assert.assertTrue("Wrong location: " + location,
location.startsWith(TestJettyHelper.getJettyURL().toString())); location.startsWith(TestJettyHelper.getJettyURL().toString()));
@ -1871,4 +1871,50 @@ public void testStoragePolicySatisfier() throws Exception {
assertTrue( assertTrue(
xAttrs.containsKey(HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY)); xAttrs.containsKey(HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY));
} }
@Test
@TestDir
@TestJetty
@TestHdfs
public void testNoRedirectWithData() throws Exception {
createHttpFSServer(false, false);
final String path = "/file";
final String username = HadoopUsersConfTestHelper.getHadoopUsers()[0];
// file creation which should not redirect
URL url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format(
"/webhdfs/v1{0}?user.name={1}&op=CREATE&data=true&noredirect=true",
path, username));
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod(HttpMethod.PUT);
conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
conn.setDoOutput(true);
conn.connect();
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
JSONObject json = (JSONObject) new JSONParser()
.parse(new InputStreamReader(conn.getInputStream()));
// get the location to write
String location = (String) json.get("Location");
Assert.assertTrue(location.contains(DataParam.NAME));
Assert.assertTrue(location.contains("CREATE"));
url = new URL(location);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod(HttpMethod.PUT);
conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
conn.setDoOutput(true);
conn.connect();
final String writeStr = "write some content";
OutputStream os = conn.getOutputStream();
os.write(writeStr.getBytes());
os.close();
// Verify that file got created
Assert.assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
json = (JSONObject) new JSONParser()
.parse(new InputStreamReader(conn.getInputStream()));
location = (String) json.get("Location");
Assert.assertEquals(TestJettyHelper.getJettyURL() + "/webhdfs/v1" + path,
location);
}
} }