big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14730. Support protobuf FileStatus in AdlFileSystem.

Browse Source
This commit is contained in:
Chris Douglas 2017-08-07 21:31:28 -07:00
parent 8d3fd81980
commit 55a181f845
4 changed files with 105 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.hadoop.fs.adl;
import com.microsoft.azure.datalake.store.DirectoryEntry;
import com.microsoft.azure.datalake.store.DirectoryEntryType;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_REPLICATION_FACTOR;
/**
* Shim class supporting linking against 2.x clients.
*/
class AdlFileStatus extends FileStatus {
private static final long serialVersionUID = 0x01fcbe5e;
private boolean hasAcl = false;
AdlFileStatus(DirectoryEntry entry, Path path, boolean hasAcl) {
this(entry, path, entry.user, entry.group, hasAcl);
}
AdlFileStatus(DirectoryEntry entry, Path path,
String owner, String group, boolean hasAcl) {
super(entry.length, DirectoryEntryType.DIRECTORY == entry.type,
ADL_REPLICATION_FACTOR, ADL_BLOCK_SIZE,
entry.lastModifiedTime.getTime(), entry.lastAccessTime.getTime(),
new AdlPermission(hasAcl, Short.parseShort(entry.permission, 8)),
owner, group, null, path);
this.hasAcl = hasAcl;
}
@Override
public boolean hasAcl() {
return hasAcl;
}
@Override
public boolean equals(Object o) {
// satisfy findbugs
return super.equals(o);
}
@Override
public int hashCode() {
// satisfy findbugs
return super.hashCode();
}
}

27
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
View File

@ -29,7 +29,6 @@
import com.microsoft.azure.datalake.store.ADLStoreClient;
import com.microsoft.azure.datalake.store.ADLStoreOptions;
import com.microsoft.azure.datalake.store.DirectoryEntry;
import com.microsoft.azure.datalake.store.DirectoryEntryType;
import com.microsoft.azure.datalake.store.IfExists;
import com.microsoft.azure.datalake.store.LatencyTracker;
import com.microsoft.azure.datalake.store.UserGroupRepresentation;
@ -606,30 +605,12 @@ private FsPermission applyUMask(FsPermission permission) {
}
private FileStatus toFileStatus(final DirectoryEntry entry, final Path f) {
boolean isDirectory = entry.type == DirectoryEntryType.DIRECTORY;
long lastModificationData = entry.lastModifiedTime.getTime();
long lastAccessTime = entry.lastAccessTime.getTime();
// set aclBit from ADLS backend response if
// ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION is true.
final boolean aclBit = aclBitStatus ? entry.aclBit : false;
FsPermission permission = new AdlPermission(aclBit,
Short.valueOf(entry.permission, 8));
String user = entry.user;
String group = entry.group;
FileStatus status;
Path p = makeQualified(f);
boolean aclBit = aclBitStatus ? entry.aclBit : false;
if (overrideOwner) {
status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR,
ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission,
userName, "hdfs", this.makeQualified(f));
} else {
status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR,
ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission,
user, group, this.makeQualified(f));
return new AdlFileStatus(entry, p, userName, "hdfs", aclBit);
}
return status;
return new AdlFileStatus(entry, p, aclBit);
}
/**

57
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
View File

@ -42,8 +42,8 @@
* org.apache.hadoop.fs.adl.live testing package.
*/
public class TestGetFileStatus extends AdlMockWebServer {
private static final Logger LOG = LoggerFactory
.getLogger(TestGetFileStatus.class);
private static final Logger LOG =
LoggerFactory.getLogger(TestGetFileStatus.class);
@Test
public void getFileStatusReturnsAsExpected()
@ -72,33 +72,30 @@ public void getFileStatusReturnsAsExpected()
fileStatus.isErasureCoded());
}
@Test
public void getFileStatusAclBit()
throws URISyntaxException, IOException {
// With ACLBIT set to true
getMockServer().enqueue(new MockResponse().setResponseCode(200)
.setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
long startTime = Time.monotonicNow();
FileStatus fileStatus = getMockAdlFileSystem()
.getFileStatus(new Path("/test1/test2"));
long endTime = Time.monotonicNow();
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertEquals(true, fileStatus.getPermission().getAclBit());
Assert.assertEquals(fileStatus.hasAcl(),
fileStatus.getPermission().getAclBit());
@Test
public void getFileStatusAclBit() throws URISyntaxException, IOException {
// With ACLBIT set to true
getMockServer().enqueue(new MockResponse().setResponseCode(200)
.setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
long startTime = Time.monotonicNow();
FileStatus fileStatus = getMockAdlFileSystem()
.getFileStatus(new Path("/test1/test2"));
long endTime = Time.monotonicNow();
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertTrue(fileStatus.hasAcl());
Assert.assertTrue(fileStatus.getPermission().getAclBit());
// With ACLBIT set to false
getMockServer().enqueue(new MockResponse().setResponseCode(200)
.setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
startTime = Time.monotonicNow();
fileStatus = getMockAdlFileSystem()
.getFileStatus(new Path("/test1/test2"));
endTime = Time.monotonicNow();
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertEquals(false, fileStatus.getPermission().getAclBit());
Assert.assertEquals(fileStatus.hasAcl(),
fileStatus.getPermission().getAclBit());
}
// With ACLBIT set to false
getMockServer().enqueue(new MockResponse().setResponseCode(200)
.setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
startTime = Time.monotonicNow();
fileStatus = getMockAdlFileSystem()
.getFileStatus(new Path("/test1/test2"));
endTime = Time.monotonicNow();
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertFalse(fileStatus.hasAcl());
Assert.assertFalse(fileStatus.getPermission().getAclBit());
}
}

8
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
View File

@ -102,7 +102,7 @@ public void listStatusOnFailure() throws IOException {
}
@Test
public void listStatusAclBit()
public void listStatusAcl()
throws URISyntaxException, IOException {
// With ACLBIT set to true
getMockServer().enqueue(new MockResponse().setResponseCode(200)
@ -115,7 +115,8 @@ public void listStatusAclBit()
LOG.debug("Time : " + (endTime - startTime));
for (int i = 0; i < ls.length; i++) {
Assert.assertTrue(ls[i].isDirectory());
Assert.assertEquals(true, ls[i].getPermission().getAclBit());
Assert.assertTrue(ls[i].hasAcl());
Assert.assertTrue(ls[i].getPermission().getAclBit());
}
// With ACLBIT set to false
@ -129,7 +130,8 @@ public void listStatusAclBit()
LOG.debug("Time : " + (endTime - startTime));
for (int i = 0; i < ls.length; i++) {
Assert.assertTrue(ls[i].isDirectory());
Assert.assertEquals(false, ls[i].getPermission().getAclBit());
Assert.assertFalse(ls[i].hasAcl());
Assert.assertFalse(ls[i].getPermission().getAclBit());
}
}
}