HADOOP-15187. Remove ADL mock test dependency on REST call invoked from Java SDK.
Contributed by Vishwajeet Dusane. (cherry picked from commit bd5ab5912564d2d687651b01f552b8e4ca8c145a)
This commit is contained in:
Steve Loughran
parent
d02e42cee4
commit
8cf88fcd1f
@ -1,102 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
|
||||
import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockWebServer;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
||||
/**
|
||||
* Mock server to simulate Adls backend calls. This infrastructure is expandable
|
||||
* to override expected server response based on the derived test functionality.
|
||||
* Common functionality to generate token information before request is send to
|
||||
* adls backend is also managed within AdlMockWebServer implementation using
|
||||
* {@link org.apache.hadoop.fs.adl.common.CustomMockTokenProvider}.
|
||||
*/
|
||||
public class AdlMockWebServer {
|
||||
// Create a MockWebServer. These are lean enough that you can create a new
|
||||
// instance for every unit test.
|
||||
private MockWebServer server = null;
|
||||
private TestableAdlFileSystem fs = null;
|
||||
private int port = 0;
|
||||
private Configuration conf = new Configuration();
|
||||
|
||||
public MockWebServer getMockServer() {
|
||||
return server;
|
||||
}
|
||||
|
||||
public TestableAdlFileSystem getMockAdlFileSystem() {
|
||||
return fs;
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public Configuration getConf() {
|
||||
return conf;
|
||||
}
|
||||
|
||||
public void setConf(Configuration conf) {
|
||||
this.conf = conf;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void preTestSetup() throws IOException, URISyntaxException {
|
||||
server = new MockWebServer();
|
||||
|
||||
// Start the server.
|
||||
server.start();
|
||||
|
||||
// Ask the server for its URL. You'll need this to make HTTP requests.
|
||||
URL baseUrl = server.getUrl("");
|
||||
port = baseUrl.getPort();
|
||||
|
||||
// Exercise your application code, which should make those HTTP requests.
|
||||
// Responses are returned in the same order that they are enqueued.
|
||||
fs = new TestableAdlFileSystem();
|
||||
|
||||
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
|
||||
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
CustomMockTokenProvider.class, AzureADTokenProvider.class);
|
||||
|
||||
URI uri = new URI("adl://localhost:" + port);
|
||||
fs.initialize(uri, conf);
|
||||
}
|
||||
|
||||
@After
|
||||
public void postTestSetup() throws IOException {
|
||||
fs.close();
|
||||
server.shutdown();
|
||||
}
|
||||
}
|
||||
@ -1,262 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.AclEntry;
|
||||
import org.apache.hadoop.fs.permission.AclEntryScope;
|
||||
import org.apache.hadoop.fs.permission.AclEntryType;
|
||||
import org.apache.hadoop.fs.permission.AclStatus;
|
||||
import org.apache.hadoop.fs.permission.FsAction;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Stub adl server and test acl data conversion within SDK and Hadoop adl
|
||||
* client.
|
||||
*/
|
||||
public class TestACLFeatures extends AdlMockWebServer {
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testModifyAclEntries() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
List<AclEntry> entries = new ArrayList<AclEntry>();
|
||||
AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
|
||||
aclEntryBuilder.setName("hadoop");
|
||||
aclEntryBuilder.setType(AclEntryType.USER);
|
||||
aclEntryBuilder.setPermission(FsAction.ALL);
|
||||
aclEntryBuilder.setScope(AclEntryScope.ACCESS);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
aclEntryBuilder.setName("hdfs");
|
||||
aclEntryBuilder.setType(AclEntryType.GROUP);
|
||||
aclEntryBuilder.setPermission(FsAction.READ_WRITE);
|
||||
aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
getMockAdlFileSystem().modifyAclEntries(new Path("/test1/test2"), entries);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem()
|
||||
.modifyAclEntries(new Path("/test1/test2"), entries);
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testRemoveAclEntriesWithOnlyUsers()
|
||||
throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
List<AclEntry> entries = new ArrayList<AclEntry>();
|
||||
AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
|
||||
aclEntryBuilder.setName("hadoop");
|
||||
aclEntryBuilder.setType(AclEntryType.USER);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem()
|
||||
.removeAclEntries(new Path("/test1/test2"), entries);
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testRemoveAclEntries() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
List<AclEntry> entries = new ArrayList<AclEntry>();
|
||||
AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
|
||||
aclEntryBuilder.setName("hadoop");
|
||||
aclEntryBuilder.setType(AclEntryType.USER);
|
||||
aclEntryBuilder.setPermission(FsAction.ALL);
|
||||
aclEntryBuilder.setScope(AclEntryScope.ACCESS);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
aclEntryBuilder.setName("hdfs");
|
||||
aclEntryBuilder.setType(AclEntryType.GROUP);
|
||||
aclEntryBuilder.setPermission(FsAction.READ_WRITE);
|
||||
aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem()
|
||||
.removeAclEntries(new Path("/test1/test2"), entries);
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testRemoveDefaultAclEntries()
|
||||
throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2"));
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2"));
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testRemoveAcl() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().removeAcl(new Path("/test1/test2"));
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem().removeAcl(new Path("/test1/test2"));
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testSetAcl() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
List<AclEntry> entries = new ArrayList<AclEntry>();
|
||||
AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
|
||||
aclEntryBuilder.setName("hadoop");
|
||||
aclEntryBuilder.setType(AclEntryType.USER);
|
||||
aclEntryBuilder.setPermission(FsAction.ALL);
|
||||
aclEntryBuilder.setScope(AclEntryScope.ACCESS);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
aclEntryBuilder.setName("hdfs");
|
||||
aclEntryBuilder.setType(AclEntryType.GROUP);
|
||||
aclEntryBuilder.setPermission(FsAction.READ_WRITE);
|
||||
aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
|
||||
entries.add(aclEntryBuilder.build());
|
||||
|
||||
getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries);
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testCheckAccess() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.ALL);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.EXECUTE);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.READ);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem()
|
||||
.access(new Path("/test1/test2"), FsAction.READ_EXECUTE);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem()
|
||||
.access(new Path("/test1/test2"), FsAction.READ_WRITE);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.NONE);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.WRITE);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem()
|
||||
.access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem()
|
||||
.access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE);
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testSetPermission() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem()
|
||||
.setPermission(new Path("/test1/test2"), FsPermission.getDefault());
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem()
|
||||
.setPermission(new Path("/test1/test2"), FsPermission.getDefault());
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testSetOwner() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200));
|
||||
getMockAdlFileSystem().setOwner(new Path("/test1/test2"), "hadoop", "hdfs");
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem()
|
||||
.setOwner(new Path("/test1/test2"), "hadoop", "hdfs");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getAclStatusAsExpected() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getGetAclStatusJSONResponse()));
|
||||
AclStatus aclStatus = getMockAdlFileSystem()
|
||||
.getAclStatus(new Path("/test1/test2"));
|
||||
Assert.assertEquals(aclStatus.getGroup(), "supergroup");
|
||||
Assert.assertEquals(aclStatus.getOwner(), "hadoop");
|
||||
Assert.assertEquals((Short) aclStatus.getPermission().toShort(),
|
||||
Short.valueOf("775", 8));
|
||||
|
||||
for (AclEntry entry : aclStatus.getEntries()) {
|
||||
if (!(entry.toString().equalsIgnoreCase("user:carla:rw-") || entry
|
||||
.toString().equalsIgnoreCase("group::r-x"))) {
|
||||
Assert.fail("Unexpected entry : " + entry.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expected=FileNotFoundException.class)
|
||||
public void getAclStatusNotExists() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(404)
|
||||
.setBody(TestADLResponseData.getFileNotFoundException()));
|
||||
|
||||
getMockAdlFileSystem().getAclStatus(new Path("/test1/test2"));
|
||||
}
|
||||
|
||||
@Test(expected=AccessControlException.class)
|
||||
public void testAclStatusDenied() throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403)
|
||||
.setBody(TestADLResponseData.getAccessControlException()));
|
||||
|
||||
getMockAdlFileSystem().getAclStatus(new Path("/test1/test2"));
|
||||
}
|
||||
}
|
||||
@ -1,196 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.adl.common.Parallelized;
|
||||
import org.apache.hadoop.fs.adl.common.TestDataForRead;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.EOFException;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Random;
|
||||
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys.READ_AHEAD_BUFFER_SIZE_KEY;
|
||||
|
||||
/**
|
||||
* This class is responsible for stress positional reads vs number of network
|
||||
* calls required by to fetch the amount of data. Test does ensure the data
|
||||
* integrity and order of the data is maintained.
|
||||
*/
|
||||
@RunWith(Parallelized.class)
|
||||
public class TestAdlRead extends AdlMockWebServer {
|
||||
|
||||
private TestDataForRead testData;
|
||||
|
||||
public TestAdlRead(TestDataForRead testData) {
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setInt(READ_AHEAD_BUFFER_SIZE_KEY, 4 * 1024);
|
||||
setConf(configuration);
|
||||
this.testData = testData;
|
||||
}
|
||||
|
||||
@Parameterized.Parameters(name = "{index}")
|
||||
public static Collection testDataForReadOperation() {
|
||||
return Arrays.asList(new Object[][] {
|
||||
|
||||
//--------------------------
|
||||
// Test Data
|
||||
//--------------------------
|
||||
{new TestDataForRead("Hello World".getBytes(), 2, 1000, true)},
|
||||
{new TestDataForRead(
|
||||
("the problem you appear to be wrestling with is that this doesn't "
|
||||
+ "display very well. ").getBytes(), 2, 1000, true)},
|
||||
{new TestDataForRead(("您的數據是寶貴的資產，以您的組織，並有當前和未來價值。由於這個原因，"
|
||||
+ "所有的數據應存儲以供將來分析。今天，這往往是不這樣做，" + "因為傳統的分析基礎架構的限制，"
|
||||
+ "像模式的預定義，存儲大數據集和不同的數據筒倉的傳播的成本。"
|
||||
+ "為了應對這一挑戰，數據湖面概念被引入作為一個企業級存儲庫來存儲所有"
|
||||
+ "類型的在一個地方收集到的數據。對於運作和探索性分析的目的，所有類型的" + "數據可以定義需求或模式之前被存儲在數據湖。")
|
||||
.getBytes(), 2, 1000, true)}, {new TestDataForRead(
|
||||
TestADLResponseData.getRandomByteArrayData(4 * 1024), 2, 10, true)},
|
||||
{new TestDataForRead(TestADLResponseData.getRandomByteArrayData(100), 2,
|
||||
1000, true)}, {new TestDataForRead(
|
||||
TestADLResponseData.getRandomByteArrayData(1 * 1024), 2, 50, true)},
|
||||
{new TestDataForRead(
|
||||
TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10,
|
||||
false)}, {new TestDataForRead(
|
||||
TestADLResponseData.getRandomByteArrayData(16 * 1024), 5, 10, false)},
|
||||
{new TestDataForRead(
|
||||
TestADLResponseData.getRandomByteArrayData(32 * 1024), 9, 10,
|
||||
false)}, {new TestDataForRead(
|
||||
TestADLResponseData.getRandomByteArrayData(64 * 1024), 17, 10,
|
||||
false)}});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEntireBytes() throws IOException, InterruptedException {
|
||||
getMockServer().setDispatcher(testData.getDispatcher());
|
||||
FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
|
||||
byte[] expectedData = new byte[testData.getActualData().length];
|
||||
int n = 0;
|
||||
int len = expectedData.length;
|
||||
int off = 0;
|
||||
while (n < len) {
|
||||
int count = in.read(expectedData, off + n, len - n);
|
||||
if (count < 0) {
|
||||
throw new EOFException();
|
||||
}
|
||||
n += count;
|
||||
}
|
||||
|
||||
Assert.assertEquals(expectedData.length, testData.getActualData().length);
|
||||
Assert.assertArrayEquals(expectedData, testData.getActualData());
|
||||
in.close();
|
||||
if (testData.isCheckOfNoOfCalls()) {
|
||||
Assert.assertEquals(testData.getExpectedNoNetworkCall(),
|
||||
getMockServer().getRequestCount());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSeekOperation() throws IOException, InterruptedException {
|
||||
getMockServer().setDispatcher(testData.getDispatcher());
|
||||
FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
|
||||
Random random = new Random();
|
||||
for (int i = 0; i < 1000; ++i) {
|
||||
int position = random.nextInt(testData.getActualData().length);
|
||||
in.seek(position);
|
||||
Assert.assertEquals(position, in.getPos());
|
||||
Assert.assertEquals(testData.getActualData()[position] & 0xFF, in.read());
|
||||
}
|
||||
in.close();
|
||||
if (testData.isCheckOfNoOfCalls()) {
|
||||
Assert.assertEquals(testData.getExpectedNoNetworkCall(),
|
||||
getMockServer().getRequestCount());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadServerCalls() throws IOException, InterruptedException {
|
||||
getMockServer().setDispatcher(testData.getDispatcher());
|
||||
FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
|
||||
byte[] expectedData = new byte[testData.getActualData().length];
|
||||
in.readFully(expectedData);
|
||||
Assert.assertArrayEquals(expectedData, testData.getActualData());
|
||||
Assert.assertEquals(testData.getExpectedNoNetworkCall(),
|
||||
getMockServer().getRequestCount());
|
||||
in.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadFully() throws IOException, InterruptedException {
|
||||
getMockServer().setDispatcher(testData.getDispatcher());
|
||||
FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
|
||||
byte[] expectedData = new byte[testData.getActualData().length];
|
||||
in.readFully(expectedData);
|
||||
Assert.assertArrayEquals(expectedData, testData.getActualData());
|
||||
|
||||
in.readFully(0, expectedData);
|
||||
Assert.assertArrayEquals(expectedData, testData.getActualData());
|
||||
|
||||
in.readFully(0, expectedData, 0, expectedData.length);
|
||||
Assert.assertArrayEquals(expectedData, testData.getActualData());
|
||||
in.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRandomPositionalReadUsingReadFully()
|
||||
throws IOException, InterruptedException {
|
||||
getMockServer().setDispatcher(testData.getDispatcher());
|
||||
FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
|
||||
ByteArrayInputStream actualData = new ByteArrayInputStream(
|
||||
testData.getActualData());
|
||||
Random random = new Random();
|
||||
for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
|
||||
int offset = random.nextInt(testData.getActualData().length);
|
||||
int length = testData.getActualData().length - offset;
|
||||
byte[] expectedData = new byte[length];
|
||||
byte[] actualDataSubset = new byte[length];
|
||||
actualData.reset();
|
||||
actualData.skip(offset);
|
||||
actualData.read(actualDataSubset, 0, length);
|
||||
|
||||
in.readFully(offset, expectedData, 0, length);
|
||||
Assert.assertArrayEquals(expectedData, actualDataSubset);
|
||||
}
|
||||
|
||||
for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
|
||||
int offset = random.nextInt(testData.getActualData().length);
|
||||
int length = random.nextInt(testData.getActualData().length - offset);
|
||||
byte[] expectedData = new byte[length];
|
||||
byte[] actualDataSubset = new byte[length];
|
||||
actualData.reset();
|
||||
actualData.skip(offset);
|
||||
actualData.read(actualDataSubset, 0, length);
|
||||
|
||||
in.readFully(offset, expectedData, 0, length);
|
||||
Assert.assertArrayEquals(expectedData, actualDataSubset);
|
||||
}
|
||||
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
@ -1,299 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.Dispatcher;
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
import com.squareup.okhttp.mockwebserver.RecordedRequest;
|
||||
import okio.Buffer;
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Random;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* This class is responsible for testing multiple threads trying to access same
|
||||
* or multiple files from the offset.
|
||||
*/
|
||||
@RunWith(Parameterized.class)
|
||||
public class TestConcurrentDataReadOperations extends AdlMockWebServer {
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(TestConcurrentDataReadOperations.class);
|
||||
private static final Object LOCK = new Object();
|
||||
private static FSDataInputStream commonHandle = null;
|
||||
private int concurrencyLevel;
|
||||
|
||||
public TestConcurrentDataReadOperations(int concurrencyLevel) {
|
||||
this.concurrencyLevel = concurrencyLevel;
|
||||
}
|
||||
|
||||
@Parameterized.Parameters(name = "{index}")
|
||||
public static Collection<?> testDataNumberOfConcurrentRun() {
|
||||
return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}});
|
||||
}
|
||||
|
||||
public static byte[] getRandomByteArrayData(int size) {
|
||||
byte[] b = new byte[size];
|
||||
Random rand = new Random();
|
||||
rand.nextBytes(b);
|
||||
return b;
|
||||
}
|
||||
|
||||
private void setDispatcher(final ArrayList<CreateTestData> testData) {
|
||||
getMockServer().setDispatcher(new Dispatcher() {
|
||||
@Override
|
||||
public MockResponse dispatch(RecordedRequest recordedRequest)
|
||||
throws InterruptedException {
|
||||
CreateTestData currentRequest = null;
|
||||
for (CreateTestData local : testData) {
|
||||
if (recordedRequest.getPath().contains(local.path.toString())) {
|
||||
currentRequest = local;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (currentRequest == null) {
|
||||
new MockResponse().setBody("Request data not found")
|
||||
.setResponseCode(501);
|
||||
}
|
||||
|
||||
if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
|
||||
return new MockResponse().setResponseCode(200).setBody(
|
||||
TestADLResponseData
|
||||
.getGetFileStatusJSONResponse(currentRequest.data.length));
|
||||
}
|
||||
|
||||
if (recordedRequest.getRequestLine().contains("op=OPEN")) {
|
||||
String request = recordedRequest.getRequestLine();
|
||||
int offset = 0;
|
||||
int byteCount = 0;
|
||||
|
||||
Pattern pattern = Pattern.compile("offset=([0-9]+)");
|
||||
Matcher matcher = pattern.matcher(request);
|
||||
if (matcher.find()) {
|
||||
LOG.debug(matcher.group(1));
|
||||
offset = Integer.parseInt(matcher.group(1));
|
||||
}
|
||||
|
||||
pattern = Pattern.compile("length=([0-9]+)");
|
||||
matcher = pattern.matcher(request);
|
||||
if (matcher.find()) {
|
||||
LOG.debug(matcher.group(1));
|
||||
byteCount = Integer.parseInt(matcher.group(1));
|
||||
}
|
||||
|
||||
Buffer buf = new Buffer();
|
||||
buf.write(currentRequest.data, offset,
|
||||
Math.min(currentRequest.data.length - offset, byteCount));
|
||||
return new MockResponse().setResponseCode(200)
|
||||
.setChunkedBody(buf, 4 * 1024 * 1024);
|
||||
}
|
||||
|
||||
return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Before
|
||||
public void resetHandle() {
|
||||
commonHandle = null;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParallelReadOnDifferentStreams()
|
||||
throws IOException, InterruptedException, ExecutionException {
|
||||
|
||||
ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
|
||||
|
||||
Random random = new Random();
|
||||
|
||||
for (int i = 0; i < concurrencyLevel; i++) {
|
||||
CreateTestData testData = new CreateTestData();
|
||||
testData
|
||||
.set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
|
||||
getRandomByteArrayData(random.nextInt(1 * 1024 * 1024)));
|
||||
createTestData.add(testData);
|
||||
}
|
||||
|
||||
setDispatcher(createTestData);
|
||||
|
||||
ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
|
||||
for (CreateTestData local : createTestData) {
|
||||
ReadTestData localReadData = new ReadTestData();
|
||||
localReadData.set(local.path, local.data, 0);
|
||||
readTestData.add(localReadData);
|
||||
}
|
||||
|
||||
runReadTest(readTestData, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParallelReadOnSameStreams()
|
||||
throws IOException, InterruptedException, ExecutionException {
|
||||
ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
|
||||
|
||||
Random random = new Random();
|
||||
|
||||
for (int i = 0; i < 1; i++) {
|
||||
CreateTestData testData = new CreateTestData();
|
||||
testData
|
||||
.set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
|
||||
getRandomByteArrayData(1024 * 1024));
|
||||
createTestData.add(testData);
|
||||
}
|
||||
|
||||
setDispatcher(createTestData);
|
||||
|
||||
ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
|
||||
ByteArrayInputStream buffered = new ByteArrayInputStream(
|
||||
createTestData.get(0).data);
|
||||
|
||||
ReadTestData readInitially = new ReadTestData();
|
||||
byte[] initialData = new byte[1024 * 1024];
|
||||
buffered.read(initialData);
|
||||
|
||||
readInitially.set(createTestData.get(0).path, initialData, 0);
|
||||
readTestData.add(readInitially);
|
||||
runReadTest(readTestData, false);
|
||||
|
||||
readTestData.clear();
|
||||
|
||||
for (int i = 0; i < concurrencyLevel * 5; i++) {
|
||||
ReadTestData localReadData = new ReadTestData();
|
||||
int offset = random.nextInt((1024 * 1024) - 1);
|
||||
int length = 1024 * 1024 - offset;
|
||||
byte[] expectedData = new byte[length];
|
||||
buffered.reset();
|
||||
buffered.skip(offset);
|
||||
buffered.read(expectedData);
|
||||
localReadData.set(createTestData.get(0).path, expectedData, offset);
|
||||
readTestData.add(localReadData);
|
||||
}
|
||||
|
||||
runReadTest(readTestData, true);
|
||||
}
|
||||
|
||||
void runReadTest(ArrayList<ReadTestData> testData, boolean useSameStream)
|
||||
throws InterruptedException, ExecutionException {
|
||||
|
||||
ExecutorService executor = Executors.newFixedThreadPool(testData.size());
|
||||
Future[] subtasks = new Future[testData.size()];
|
||||
|
||||
for (int i = 0; i < testData.size(); i++) {
|
||||
subtasks[i] = executor.submit(
|
||||
new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path,
|
||||
testData.get(i).offset, useSameStream));
|
||||
}
|
||||
|
||||
executor.shutdown();
|
||||
|
||||
// wait until all tasks are finished
|
||||
executor.awaitTermination(120, TimeUnit.SECONDS);
|
||||
|
||||
for (int i = 0; i < testData.size(); ++i) {
|
||||
Assert.assertTrue((Boolean) subtasks[i].get());
|
||||
}
|
||||
}
|
||||
|
||||
class ReadTestData {
|
||||
private Path path;
|
||||
private byte[] data;
|
||||
private int offset;
|
||||
|
||||
public void set(Path filePath, byte[] dataToBeRead, int fromOffset) {
|
||||
this.path = filePath;
|
||||
this.data = dataToBeRead;
|
||||
this.offset = fromOffset;
|
||||
}
|
||||
}
|
||||
|
||||
class CreateTestData {
|
||||
private Path path;
|
||||
private byte[] data;
|
||||
|
||||
public void set(Path filePath, byte[] dataToBeWritten) {
|
||||
this.path = filePath;
|
||||
this.data = dataToBeWritten;
|
||||
}
|
||||
}
|
||||
|
||||
class ReadConcurrentRunnable implements Callable<Boolean> {
|
||||
private Path path;
|
||||
private int offset;
|
||||
private byte[] expectedData;
|
||||
private boolean useSameStream;
|
||||
|
||||
public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset,
|
||||
boolean useSameStream) {
|
||||
this.path = path;
|
||||
this.offset = offset;
|
||||
this.expectedData = expectedData;
|
||||
this.useSameStream = useSameStream;
|
||||
}
|
||||
|
||||
public Boolean call() throws IOException {
|
||||
try {
|
||||
FSDataInputStream in;
|
||||
if (useSameStream) {
|
||||
synchronized (LOCK) {
|
||||
if (commonHandle == null) {
|
||||
commonHandle = getMockAdlFileSystem().open(path);
|
||||
}
|
||||
in = commonHandle;
|
||||
}
|
||||
} else {
|
||||
in = getMockAdlFileSystem().open(path);
|
||||
}
|
||||
|
||||
byte[] actualData = new byte[expectedData.length];
|
||||
in.readFully(offset, actualData);
|
||||
Assert.assertArrayEquals("Path :" + path.toString() + " did not match.",
|
||||
expectedData, actualData);
|
||||
if (!useSameStream) {
|
||||
in.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,140 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
|
||||
|
||||
/**
|
||||
* Test access token provider behaviour with custom token provider and for token
|
||||
* provider cache is enabled.
|
||||
*/
|
||||
@RunWith(Parameterized.class)
|
||||
public class TestCustomTokenProvider extends AdlMockWebServer {
|
||||
private static final long TEN_MINUTES_IN_MILIS = 600000;
|
||||
private int backendCallCount;
|
||||
private int expectedCallbackToAccessToken;
|
||||
private TestableAdlFileSystem[] fileSystems;
|
||||
private Class typeOfTokenProviderClass;
|
||||
private long expiryFromNow;
|
||||
private int fsObjectCount;
|
||||
|
||||
public TestCustomTokenProvider(Class typeOfTokenProviderClass,
|
||||
long expiryFromNow, int fsObjectCount, int backendCallCount,
|
||||
int expectedCallbackToAccessToken)
|
||||
throws IllegalAccessException, InstantiationException, URISyntaxException,
|
||||
IOException {
|
||||
this.typeOfTokenProviderClass = typeOfTokenProviderClass;
|
||||
this.expiryFromNow = expiryFromNow;
|
||||
this.fsObjectCount = fsObjectCount;
|
||||
this.backendCallCount = backendCallCount;
|
||||
this.expectedCallbackToAccessToken = expectedCallbackToAccessToken;
|
||||
}
|
||||
|
||||
@Parameterized.Parameters(name = "{index}")
|
||||
public static Collection testDataForTokenProvider() {
|
||||
return Arrays.asList(new Object[][] {
|
||||
// Data set in order
|
||||
// INPUT - CustomTokenProvider class to load
|
||||
// INPUT - expiry time in milis. Subtract from current time
|
||||
// INPUT - No. of FileSystem object
|
||||
// INPUT - No. of backend calls per FileSystem object
|
||||
// EXPECTED - Number of callbacks to get token after test finished.
|
||||
{CustomMockTokenProvider.class, 0, 1, 1, 1},
|
||||
{CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 1, 1, 1},
|
||||
{CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 2, 1, 2},
|
||||
{CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 10, 10, 10}});
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicitly invoked init so that base class mock server is setup before
|
||||
* test data initialization is done.
|
||||
*
|
||||
* @throws IOException
|
||||
* @throws URISyntaxException
|
||||
*/
|
||||
public void init() throws IOException, URISyntaxException {
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
|
||||
TokenProviderType.Custom);
|
||||
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
typeOfTokenProviderClass.getName());
|
||||
fileSystems = new TestableAdlFileSystem[fsObjectCount];
|
||||
URI uri = new URI("adl://localhost:" + getPort());
|
||||
|
||||
for (int i = 0; i < fsObjectCount; ++i) {
|
||||
fileSystems[i] = new TestableAdlFileSystem();
|
||||
fileSystems[i].initialize(uri, configuration);
|
||||
|
||||
((CustomMockTokenProvider) fileSystems[i].getAzureTokenProvider())
|
||||
.setExpiryTimeInMillisAfter(expiryFromNow);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCustomTokenManagement()
|
||||
throws IOException, URISyntaxException {
|
||||
int accessTokenCallbackDuringExec = 0;
|
||||
init();
|
||||
for (TestableAdlFileSystem tfs : fileSystems) {
|
||||
for (int i = 0; i < backendCallCount; ++i) {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getGetFileStatusJSONResponse()));
|
||||
FileStatus fileStatus = tfs.getFileStatus(new Path("/test1/test2"));
|
||||
Assert.assertTrue(fileStatus.isFile());
|
||||
Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" +
|
||||
getMockServer().getPort() + "/test1/test2",
|
||||
fileStatus.getPath().toString());
|
||||
Assert.assertEquals(4194304, fileStatus.getLen());
|
||||
Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize());
|
||||
Assert.assertEquals(1, fileStatus.getReplication());
|
||||
Assert
|
||||
.assertEquals(new FsPermission("777"), fileStatus.getPermission());
|
||||
Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
|
||||
Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
|
||||
}
|
||||
|
||||
accessTokenCallbackDuringExec += ((CustomMockTokenProvider) tfs
|
||||
.getAzureTokenProvider()).getAccessTokenRequestCount();
|
||||
}
|
||||
|
||||
Assert.assertEquals(expectedCallbackToAccessToken,
|
||||
accessTokenCallbackDuringExec);
|
||||
}
|
||||
}
|
||||
@ -1,102 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.util.Time;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
|
||||
|
||||
/**
|
||||
* This class is responsible for testing local getFileStatus implementation
|
||||
* to cover correct parsing of successful and error JSON response
|
||||
* from the server.
|
||||
* Adls GetFileStatus operation is in detail covered in
|
||||
* org.apache.hadoop.fs.adl.live testing package.
|
||||
*/
|
||||
public class TestGetFileStatus extends AdlMockWebServer {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(TestGetFileStatus.class);
|
||||
|
||||
@Test
|
||||
public void getFileStatusReturnsAsExpected()
|
||||
throws URISyntaxException, IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getGetFileStatusJSONResponse()));
|
||||
long startTime = Time.monotonicNow();
|
||||
Path path = new Path("/test1/test2");
|
||||
FileStatus fileStatus = getMockAdlFileSystem().getFileStatus(path);
|
||||
long endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
Assert.assertTrue(fileStatus.isFile());
|
||||
Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" +
|
||||
getMockServer().getPort() + "/test1/test2",
|
||||
fileStatus.getPath().toString());
|
||||
Assert.assertEquals(4194304, fileStatus.getLen());
|
||||
Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize());
|
||||
Assert.assertEquals(1, fileStatus.getReplication());
|
||||
Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission());
|
||||
Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
|
||||
Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
|
||||
Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl());
|
||||
Assert.assertFalse(path + " should not be encrypted!",
|
||||
fileStatus.isEncrypted());
|
||||
Assert.assertFalse(path + " should not be erasure coded!",
|
||||
fileStatus.isErasureCoded());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getFileStatusAclBit() throws URISyntaxException, IOException {
|
||||
// With ACLBIT set to true
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
|
||||
long startTime = Time.monotonicNow();
|
||||
FileStatus fileStatus = getMockAdlFileSystem()
|
||||
.getFileStatus(new Path("/test1/test2"));
|
||||
long endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
Assert.assertTrue(fileStatus.isFile());
|
||||
Assert.assertTrue(fileStatus.hasAcl());
|
||||
Assert.assertTrue(fileStatus.getPermission().getAclBit());
|
||||
|
||||
// With ACLBIT set to false
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
|
||||
startTime = Time.monotonicNow();
|
||||
fileStatus = getMockAdlFileSystem()
|
||||
.getFileStatus(new Path("/test1/test2"));
|
||||
endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
Assert.assertTrue(fileStatus.isFile());
|
||||
Assert.assertFalse(fileStatus.hasAcl());
|
||||
Assert.assertFalse(fileStatus.getPermission().getAclBit());
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,137 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.util.Time;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
/**
|
||||
* This class is responsible for testing local listStatus implementation to
|
||||
* cover correct parsing of successful and error JSON response from the server.
|
||||
* Adls ListStatus functionality is in detail covered in
|
||||
* org.apache.hadoop.fs.adl.live testing package.
|
||||
*/
|
||||
public class TestListStatus extends AdlMockWebServer {
|
||||
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(TestListStatus.class);
|
||||
|
||||
@Test
|
||||
public void listStatusReturnsAsExpected() throws IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getListFileStatusJSONResponse(10)));
|
||||
long startTime = Time.monotonicNow();
|
||||
FileStatus[] ls = getMockAdlFileSystem()
|
||||
.listStatus(new Path("/test1/test2"));
|
||||
long endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
Assert.assertEquals(10, ls.length);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getListFileStatusJSONResponse(200)));
|
||||
startTime = Time.monotonicNow();
|
||||
ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
|
||||
endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
Assert.assertEquals(200, ls.length);
|
||||
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getListFileStatusJSONResponse(2048)));
|
||||
startTime = Time.monotonicNow();
|
||||
ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
|
||||
endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
Assert.assertEquals(2048, ls.length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void listStatusOnFailure() throws IOException {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(403).setBody(
|
||||
TestADLResponseData.getErrorIllegalArgumentExceptionJSONResponse()));
|
||||
FileStatus[] ls = null;
|
||||
long startTime = Time.monotonicNow();
|
||||
try {
|
||||
ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
|
||||
} catch (IOException e) {
|
||||
Assert.assertTrue(e.getMessage().contains("Invalid"));
|
||||
}
|
||||
long endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
|
||||
// SDK may increase number of retry attempts before error is propagated
|
||||
// to caller. Adding max 10 error responses in the queue to align with SDK.
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(500).setBody(
|
||||
TestADLResponseData.getErrorInternalServerExceptionJSONResponse()));
|
||||
}
|
||||
|
||||
startTime = Time.monotonicNow();
|
||||
try {
|
||||
ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
|
||||
} catch (IOException e) {
|
||||
Assert.assertTrue(e.getMessage().contains("Internal Server Error"));
|
||||
}
|
||||
endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void listStatusAcl()
|
||||
throws URISyntaxException, IOException {
|
||||
// With ACLBIT set to true
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getListFileStatusJSONResponse(true)));
|
||||
FileStatus[] ls = null;
|
||||
long startTime = Time.monotonicNow();
|
||||
ls = getMockAdlFileSystem()
|
||||
.listStatus(new Path("/test1/test2"));
|
||||
long endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
for (int i = 0; i < ls.length; i++) {
|
||||
Assert.assertTrue(ls[i].isDirectory());
|
||||
Assert.assertTrue(ls[i].hasAcl());
|
||||
Assert.assertTrue(ls[i].getPermission().getAclBit());
|
||||
}
|
||||
|
||||
// With ACLBIT set to false
|
||||
ls = null;
|
||||
getMockServer().enqueue(new MockResponse().setResponseCode(200)
|
||||
.setBody(TestADLResponseData.getListFileStatusJSONResponse(false)));
|
||||
startTime = Time.monotonicNow();
|
||||
ls = getMockAdlFileSystem()
|
||||
.listStatus(new Path("/test1/test2"));
|
||||
endTime = Time.monotonicNow();
|
||||
LOG.debug("Time : " + (endTime - startTime));
|
||||
for (int i = 0; i < ls.length; i++) {
|
||||
Assert.assertTrue(ls[i].isDirectory());
|
||||
Assert.assertFalse(ls[i].hasAcl());
|
||||
Assert.assertFalse(ls[i].getPermission().getAclBit());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl;
|
||||
|
||||
/**
|
||||
* Mock adl file storage subclass to mock adl storage on local http service.
|
||||
*/
|
||||
public class TestableAdlFileSystem extends AdlFileSystem {
|
||||
@Override
|
||||
protected String getTransportScheme() {
|
||||
return "http";
|
||||
}
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl.common;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Supporting class to hold expected MockResponse object along with parameters
|
||||
* for validation in test methods.
|
||||
*/
|
||||
public class ExpectedResponse {
|
||||
private MockResponse response;
|
||||
private ArrayList<String> expectedQueryParameters = new ArrayList<String>();
|
||||
private int expectedBodySize;
|
||||
private String httpRequestType;
|
||||
|
||||
public int getExpectedBodySize() {
|
||||
return expectedBodySize;
|
||||
}
|
||||
|
||||
public String getHttpRequestType() {
|
||||
return httpRequestType;
|
||||
}
|
||||
|
||||
public ArrayList<String> getExpectedQueryParameters() {
|
||||
return expectedQueryParameters;
|
||||
}
|
||||
|
||||
public MockResponse getResponse() {
|
||||
return response;
|
||||
}
|
||||
|
||||
ExpectedResponse set(MockResponse mockResponse) {
|
||||
this.response = mockResponse;
|
||||
return this;
|
||||
}
|
||||
|
||||
ExpectedResponse addExpectedQueryParam(String param) {
|
||||
expectedQueryParameters.add(param);
|
||||
return this;
|
||||
}
|
||||
|
||||
ExpectedResponse addExpectedBodySize(int bodySize) {
|
||||
this.expectedBodySize = bodySize;
|
||||
return this;
|
||||
}
|
||||
|
||||
ExpectedResponse addExpectedHttpRequestType(String expectedHttpRequestType) {
|
||||
this.httpRequestType = expectedHttpRequestType;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -1,122 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.adl.common;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.Dispatcher;
|
||||
import com.squareup.okhttp.mockwebserver.MockResponse;
|
||||
import com.squareup.okhttp.mockwebserver.RecordedRequest;
|
||||
import okio.Buffer;
|
||||
import org.apache.hadoop.fs.adl.TestADLResponseData;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Supporting class for mock test to validate Adls read operation.
|
||||
*/
|
||||
public class TestDataForRead {
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(TestDataForRead.class);
|
||||
|
||||
private byte[] actualData;
|
||||
private ArrayList<ExpectedResponse> responses;
|
||||
private Dispatcher dispatcher;
|
||||
private int intensityOfTest;
|
||||
private boolean checkOfNoOfCalls;
|
||||
private int expectedNoNetworkCall;
|
||||
|
||||
public TestDataForRead(final byte[] actualData, int expectedNoNetworkCall,
|
||||
int intensityOfTest, boolean checkOfNoOfCalls) {
|
||||
|
||||
this.checkOfNoOfCalls = checkOfNoOfCalls;
|
||||
this.actualData = actualData;
|
||||
responses = new ArrayList<ExpectedResponse>();
|
||||
this.expectedNoNetworkCall = expectedNoNetworkCall;
|
||||
this.intensityOfTest = intensityOfTest;
|
||||
|
||||
dispatcher = new Dispatcher() {
|
||||
@Override
|
||||
public MockResponse dispatch(RecordedRequest recordedRequest)
|
||||
throws InterruptedException {
|
||||
|
||||
if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
|
||||
return new MockResponse().setResponseCode(200).setBody(
|
||||
TestADLResponseData
|
||||
.getGetFileStatusJSONResponse(actualData.length));
|
||||
}
|
||||
|
||||
if (recordedRequest.getRequestLine().contains("op=OPEN")) {
|
||||
String request = recordedRequest.getRequestLine();
|
||||
int offset = 0;
|
||||
int byteCount = 0;
|
||||
|
||||
Pattern pattern = Pattern.compile("offset=([0-9]+)");
|
||||
Matcher matcher = pattern.matcher(request);
|
||||
if (matcher.find()) {
|
||||
LOG.debug(matcher.group(1));
|
||||
offset = Integer.parseInt(matcher.group(1));
|
||||
}
|
||||
|
||||
pattern = Pattern.compile("length=([0-9]+)");
|
||||
matcher = pattern.matcher(request);
|
||||
if (matcher.find()) {
|
||||
LOG.debug(matcher.group(1));
|
||||
byteCount = Integer.parseInt(matcher.group(1));
|
||||
}
|
||||
|
||||
Buffer buf = new Buffer();
|
||||
buf.write(actualData, offset,
|
||||
Math.min(actualData.length - offset, byteCount));
|
||||
return new MockResponse().setResponseCode(200)
|
||||
.setChunkedBody(buf, 4 * 1024 * 1024);
|
||||
}
|
||||
|
||||
return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public boolean isCheckOfNoOfCalls() {
|
||||
return checkOfNoOfCalls;
|
||||
}
|
||||
|
||||
public int getExpectedNoNetworkCall() {
|
||||
return expectedNoNetworkCall;
|
||||
}
|
||||
|
||||
public int getIntensityOfTest() {
|
||||
return intensityOfTest;
|
||||
}
|
||||
|
||||
public byte[] getActualData() {
|
||||
return actualData;
|
||||
}
|
||||
|
||||
public ArrayList<ExpectedResponse> getResponses() {
|
||||
return responses;
|
||||
}
|
||||
|
||||
public Dispatcher getDispatcher() {
|
||||
return dispatcher;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user