big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12105. Ozone: listVolumes doesn't work from ozone commandline. Contributed by Yiqun Lin.

Browse Source
This commit is contained in:
Weiwei Yang 2017-07-10 10:24:22 +08:00 committed by Owen O'Malley
parent 7afddca9d5
commit ea5e24c925
3 changed files with 132 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
View File

@ -204,24 +204,26 @@ public OzoneVolume getVolume(String volumeName) throws OzoneException {
* List all the volumes owned by the user or Owned by the user specified in
* the behalf of string.
*
* @param onBehalfOf - User Name of the user if it is not the caller. for
* example, an admin wants to list some other users
* volumes.
* @param prefix - Return only volumes that match this prefix.
* @param maxKeys - Maximum number of results to return, if the result set
* is smaller than requested size, it means that list is
* complete.
* @param prevKey - The last key that client got, server will continue
* returning results from that point.
* @param onBehalfOf
* User Name of the user if it is not the caller. for example,
* an admin wants to list some other users volumes.
* @param prefix
* Return only volumes that match this prefix.
* @param maxKeys
* Maximum number of results to return, if the result set
* is smaller than requested size, it means that list is
* complete.
* @param startVolume
* The previous volume name.
* @return List of Volumes
* @throws OzoneException
*/
public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
maxKeys, OzoneVolume prevKey) throws OzoneException {
public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
int maxKeys, String startVolume) throws OzoneException {
HttpGet httpGet = null;
try (CloseableHttpClient httpClient = newHttpClient()) {
URIBuilder builder = new URIBuilder(endPointURI);
if (prefix != null) {
if (!Strings.isNullOrEmpty(prefix)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
}
@ -230,9 +232,9 @@ public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
.toString(maxKeys));
}
if (prevKey != null) {
if (!Strings.isNullOrEmpty(startVolume)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY,
prevKey.getOwnerName() + "/" + prevKey.getVolumeName());
startVolume);
}
builder.setPath("/").build();
@ -249,6 +251,33 @@ public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
}
}
/**
* List all the volumes owned by the user or Owned by the user specified in
* the behalf of string.
*
* @param onBehalfOf - User Name of the user if it is not the caller. for
* example, an admin wants to list some other users
* volumes.
* @param prefix - Return only volumes that match this prefix.
* @param maxKeys - Maximum number of results to return, if the result set
* is smaller than requested size, it means that list is
* complete.
* @param prevKey - The last key that client got, server will continue
* returning results from that point.
* @return List of Volumes
* @throws OzoneException
*/
public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
int maxKeys, OzoneVolume prevKey) throws OzoneException {
String volumeName = null;
if (prevKey != null) {
volumeName = prevKey.getVolumeName();
}
return listVolumes(onBehalfOf, prefix, maxKeys, volumeName);
}
/**
* List volumes of the current user or if onBehalfof is not null lists volume
* owned by that user. You need admin privilege to read other users volume
@ -260,7 +289,8 @@ public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
*/
public List<OzoneVolume> listVolumes(String onBehalfOf)
throws OzoneException {
return listVolumes(onBehalfOf, null, 1000, null);
return listVolumes(onBehalfOf, null,
Integer.parseInt(Header.OZONE_DEFAULT_LIST_SIZE), StringUtils.EMPTY);
}
/**

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
View File

@ -55,6 +55,32 @@ protected void execute(CommandLine cmd)
"Incorrect call : listVolume is missing");
}
int maxKeys = 0;
if (cmd.hasOption(Shell.LIST_LENGTH)) {
String length = cmd.getOptionValue(Shell.LIST_LENGTH);
try {
maxKeys = Integer.parseInt(length);
} catch (NumberFormatException nfe) {
throw new OzoneRestClientException(
"Invalid max key length, the vaule should be digital.");
}
if (maxKeys <= 0) {
throw new OzoneRestClientException(
"Invalid max key length, the vaule should be a positive number.");
}
}
String startVolume = null;
if (cmd.hasOption(Shell.START)) {
startVolume = cmd.getOptionValue(Shell.START);
}
String prefix = null;
if (cmd.hasOption(Shell.PREFIX)) {
prefix = cmd.getOptionValue(Shell.PREFIX);
}
String ozoneURIString = cmd.getOptionValue(Shell.LIST_VOLUME);
URI ozoneURI = verifyURI(ozoneURIString);
@ -62,11 +88,6 @@ protected void execute(CommandLine cmd)
rootName = "hdfs";
}
if (!cmd.hasOption(Shell.USER)) {
throw new OzoneRestClientException(
"User name is needed in listVolume call.");
}
if (cmd.hasOption(Shell.USER)) {
userName = cmd.getOptionValue(Shell.USER);
} else {
@ -80,7 +101,8 @@ protected void execute(CommandLine cmd)
client.setUserAuth(userName);
}
List<OzoneVolume> volumes = client.listVolumes(userName);
List<OzoneVolume> volumes = client.listVolumes(userName, prefix, maxKeys,
startVolume);
if (volumes != null) {
if (cmd.hasOption(Shell.VERBOSE)) {
System.out.printf("Found : %d volumes for user : %s %n", volumes.size(),

62
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -20,11 +20,13 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
@ -45,6 +47,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -80,7 +83,7 @@ public static void init() throws Exception {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
@ -123,8 +126,9 @@ public void testCreateDuplicateVolume() throws OzoneException {
client.createVolume("testvol", "bilbo", "100TB");
assertFalse(true);
} catch (OzoneException ex) {
// OZone will throw saying volume already exists
assertEquals(ex.getShortMessage(),"volumeAlreadyExists");
// Ozone will throw saying volume already exists
GenericTestUtils.assertExceptionContains(
Status.VOLUME_ALREADY_EXISTS.toString(), ex);
}
}
@ -224,6 +228,58 @@ public void testListAllVolumes() throws OzoneException, IOException {
Assert.assertEquals(volCount / step , pagecount);
}
@Test
public void testListVolumes() throws OzoneException, IOException {
final int volCount = 20;
final String user1 = "test-user-a";
final String user2 = "test-user-b";
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
// Create 20 volumes, 10 for user1 and another 10 for user2.
for (int x = 0; x < volCount; x++) {
String volumeName;
String userName;
if (x % 2 == 0) {
// create volume [test-vol0, test-vol2, ..., test-vol18] for user1
userName = user1;
volumeName = "test-vol" + x;
} else {
// create volume [test-vol1, test-vol3, ..., test-vol19] for user2
userName = user2;
volumeName = "test-vol" + x;
}
OzoneVolume vol = client.createVolume(volumeName, userName, "100TB");
assertNotNull(vol);
}
// list all the volumes belong to user1
List<OzoneVolume> volumeList = client.listVolumes(user1,
null, 100, StringUtils.EMPTY);
assertEquals(10, volumeList.size());
volumeList.stream()
.filter(item -> item.getOwnerName().equals(user1))
.collect(Collectors.toList());
// test max key parameter of listing volumes
volumeList = client.listVolumes(user1, null, 2, StringUtils.EMPTY);
assertEquals(2, volumeList.size());
// test prefix parameter of listing volumes
volumeList = client.listVolumes(user1, "test-vol10", 100,
StringUtils.EMPTY);
assertTrue(volumeList.size() == 1
&& volumeList.get(0).getVolumeName().equals("test-vol10"));
volumeList = client.listVolumes(user1, "test-vol1",
100, StringUtils.EMPTY);
assertEquals(5, volumeList.size());
// test start key parameter of listing volumes
volumeList = client.listVolumes(user2, null, 100, "test-vol17");
assertEquals(2, volumeList.size());
}
/**
* Returns a list of mocked {@link CloseableHttpClient} used for testing.
* The mocked client replaces the actual calls in