big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone. Contributed by Anu Engineer.

Browse Source
This commit is contained in:
cnauroth 2016-02-22 17:18:49 -08:00 committed by Owen O'Malley
parent f6b9c0f8a1
commit f6ef841f74
4 changed files with 333 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt
View File

@ -27,3 +27,9 @@
HDFS-8717. OzoneHandler : Add common bucket objects. (Anu Engineer via
Arpit Agarwal)
HDFS-8757. OzoneHandler : Add localStorageHandler support for Buckets.
(Anu Engineer)
HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone.
(Anu Engineer via cnauroth)

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.ozone.web.exceptions.OzoneExceptionMapper;
import org.apache.hadoop.ozone.web.handlers.BucketHandler;
import org.apache.hadoop.ozone.web.handlers.VolumeHandler;
@ -38,6 +39,7 @@ public Set<Class<?>> getClasses() {
HashSet<Class<?>> set = new HashSet<>();
set.add(BucketHandler.class);
set.add(VolumeHandler.class);
set.add(OzoneExceptionMapper.class);
return set;
}

37
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.web.exceptions;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
/**
* Class the represents various errors returned by the
* Object Layer.
*/
public class OzoneExceptionMapper implements ExceptionMapper<OzoneException> {
@Override
public Response toResponse(OzoneException exception) {
return Response.status((int)exception.getHttpCode())
.entity(exception.toJsonString()).build();
}
}

288
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java Normal file
View File

@ -0,0 +1,288 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import static java.net.HttpURLConnection.HTTP_CREATED;
import static org.junit.Assert.assertEquals;
public class TestOzoneVolumes {
static MiniDFSCluster cluster = null;
static int port = 0;
/**
* Create a MiniDFSCluster for testing.
* <p>
* Ozone is made active by setting DFS_OBJECTSTORE_ENABLED_KEY = true and
* DFS_STORAGE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
* emulate Ozone backend.
*
* @throws IOException
*/
@BeforeClass
public static void init() throws IOException {
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
String path = p.getPath();
path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT,
OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path);
conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local");
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
}
/**
* shutdown MiniDFSCluster
*/
@AfterClass
public static void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
}
/**
* Creates Volumes on Ozone Store.
*
* @throws IOException
*/
@Test
public void testCreateVolumes() throws IOException {
SimpleDateFormat format =
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
HttpClient client = new DefaultHttpClient();
String volumeName = OzoneUtils.getRequestID().toLowerCase();
try {
HttpPost httppost = new HttpPost(
String.format("http://localhost:%d/%s", port, volumeName));
httppost.addHeader(Header.OZONE_VERSION_HEADER,
Header.OZONE_V1_VERSION_HEADER);
httppost.addHeader(HttpHeaders.DATE,
format.format(new Date(Time.monotonicNow())));
httppost.addHeader(HttpHeaders.AUTHORIZATION,
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
HttpResponse response = client.execute(httppost);
assertEquals(response.toString(), HTTP_CREATED,
response.getStatusLine().getStatusCode());
} finally {
client.getConnectionManager().shutdown();
}
}
/**
* Create Volumes with Quota.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithQuota() throws IOException {
SimpleDateFormat format =
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
HttpClient client = new DefaultHttpClient();
String volumeName = OzoneUtils.getRequestID().toLowerCase();
try {
HttpPost httppost = new HttpPost(
String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
httppost.addHeader(Header.OZONE_VERSION_HEADER,
Header.OZONE_V1_VERSION_HEADER);
httppost.addHeader(HttpHeaders.DATE,
format.format(new Date(Time.monotonicNow())));
httppost.addHeader(HttpHeaders.AUTHORIZATION,
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
HttpResponse response = client.execute(httppost);
assertEquals(response.toString(), HTTP_CREATED,
response.getStatusLine().getStatusCode());
} finally {
client.getConnectionManager().shutdown();
}
}
/**
* Create Volumes with Invalid Quota.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithInvalidQuota() throws IOException {
SimpleDateFormat format =
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
HttpClient client = new DefaultHttpClient();
String volumeName = OzoneUtils.getRequestID().toLowerCase();
try {
HttpPost httppost = new HttpPost(
String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
httppost.addHeader(Header.OZONE_VERSION_HEADER,
Header.OZONE_V1_VERSION_HEADER);
httppost.addHeader(HttpHeaders.DATE,
format.format(new Date(Time.monotonicNow())));
httppost.addHeader(HttpHeaders.AUTHORIZATION,
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
HttpResponse response = client.execute(httppost);
assertEquals(response.toString(), ErrorTable.MALFORMED_QUOTA
.getHttpCode(),
response.getStatusLine().getStatusCode());
} finally {
client.getConnectionManager().shutdown();
}
}
/**
* To create a volume a user name must be specified using OZONE_USER header.
* This test verifies that we get an error in case we call without a OZONE
* user name.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithInvalidUser() throws IOException {
SimpleDateFormat format =
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
HttpClient client = new DefaultHttpClient();
String volumeName = OzoneUtils.getRequestID().toLowerCase();
try {
HttpPost httppost = new HttpPost(
String.format("http://localhost:%d/%s?quota=1TB", port, volumeName));
httppost.addHeader(Header.OZONE_VERSION_HEADER,
Header.OZONE_V1_VERSION_HEADER);
httppost.addHeader(HttpHeaders.DATE,
format.format(new Date(Time.monotonicNow())));
httppost.addHeader(HttpHeaders.AUTHORIZATION,
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
HttpResponse response = client.execute(httppost);
assertEquals(response.toString(), ErrorTable.USER_NOT_FOUND.getHttpCode(),
response.getStatusLine().getStatusCode());
} finally {
client.getConnectionManager().shutdown();
}
}
/**
* Only Admins can create volumes in Ozone. This test uses simple userauth as
* backend and hdfs and root are admin users in the simple backend.
* <p>
* This test tries to create a volume as user bilbo.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithOutAdminRights() throws IOException {
SimpleDateFormat format =
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
HttpClient client = new DefaultHttpClient();
String volumeName = OzoneUtils.getRequestID().toLowerCase();
try {
HttpPost httppost = new HttpPost(
String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
httppost.addHeader(Header.OZONE_VERSION_HEADER,
Header.OZONE_V1_VERSION_HEADER);
httppost.addHeader(HttpHeaders.DATE,
format.format(new Date(Time.monotonicNow())));
httppost.addHeader(HttpHeaders.AUTHORIZATION,
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
"bilbo"); // This is not a root user in Simple
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
HttpResponse response = client.execute(httppost);
assertEquals(response.toString(), ErrorTable.ACCESS_DENIED.getHttpCode(),
response.getStatusLine().getStatusCode());
} finally {
client.getConnectionManager().shutdown();
}
}
/**
* Create a bunch of volumes in a loop
*
* @throws IOException
*/
@Test
public void testCreateVolumesInLoop() throws IOException {
SimpleDateFormat format =
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
for (int x = 0; x < 1000; x++) {
HttpClient client = new DefaultHttpClient();
String volumeName = OzoneUtils.getRequestID().toLowerCase();
String userName = OzoneUtils.getRequestID().toLowerCase();
HttpPost httppost = new HttpPost(
String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
httppost.addHeader(Header.OZONE_VERSION_HEADER,
Header.OZONE_V1_VERSION_HEADER);
httppost.addHeader(HttpHeaders.DATE,
format.format(new Date(Time.monotonicNow())));
httppost.addHeader(HttpHeaders.AUTHORIZATION,
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
httppost.addHeader(Header.OZONE_USER, userName);
HttpResponse response = client.execute(httppost);
assertEquals(response.toString(), HTTP_CREATED,
response.getStatusLine().getStatusCode());
client.getConnectionManager().shutdown();
}
}
}