big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-8545. Filesystem Implementation for OpenStack Swift

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1526854 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Steve Loughran 2013-09-27 11:32:53 +00:00
parent 96c11fe602
commit 3caca924bc
93 changed files with 14397 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -289,6 +289,9 @@ Release 2.3.0 - UNRELEASED
NEW FEATURES
HADOOP-8545. Filesystem Implementation for OpenStack Swift
(Dmitry Mezhensky, David Dobbins, Stevel via stevel)
IMPROVEMENTS
HADOOP-9784. Add a builder for HttpServer. (Junping Du via llu)

18
hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
View File

@ -348,4 +348,20 @@
<Method name="waitForServiceToStop" />
<Bug code="JLM" />
</Match>
</FindBugsFilter>
<!--
OpenStack Swift FS module -closes streams in a different method
from where they are opened.
-->
<Match>
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
<Method name="uploadFileAttempt"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
</Match>
<Match>
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
<Method name="uploadFilePartAttempt"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
</Match>
</FindBugsFilter>

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java
View File

@ -22,7 +22,9 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/** Stream that permits seeking. */
/**
* Stream that permits seeking.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface Seekable {

5
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -500,6 +500,11 @@
</description>
</property>
<property>
<name>fs.swift.impl</name>
<value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
<description>The implementation class of the OpenStack Swift Filesystem</description>
</property>
<property>
<name>fs.automatic.close</name>

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java
View File

@ -944,14 +944,20 @@ public void testRenameDirectoryToNonExistentParent() throws Exception {
rename(src, dst, false, true, false, Rename.NONE);
Assert.fail("Expected exception was not thrown");
} catch (IOException e) {
Assert.assertTrue(unwrapException(e) instanceof FileNotFoundException);
IOException ioException = unwrapException(e);
if (!(ioException instanceof FileNotFoundException)) {
throw ioException;
}
}
try {
rename(src, dst, false, true, false, Rename.OVERWRITE);
Assert.fail("Expected exception was not thrown");
} catch (IOException e) {
Assert.assertTrue(unwrapException(e) instanceof FileNotFoundException);
IOException ioException = unwrapException(e);
if (!(ioException instanceof FileNotFoundException)) {
throw ioException;
}
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
View File

@ -291,7 +291,7 @@ public void testOverwrite() throws IOException {
public void testWriteInNonExistentDirectory() throws IOException {
Path path = path("/test/hadoop/file");
assertFalse("Parent doesn't exist", fs.exists(path.getParent()));
assertFalse("Parent exists", fs.exists(path.getParent()));
createFile(path);
assertTrue("Exists", fs.exists(path));
@ -301,7 +301,7 @@ public void testWriteInNonExistentDirectory() throws IOException {
public void testDeleteNonExistentFile() throws IOException {
Path path = path("/test/hadoop/file");
assertFalse("Doesn't exist", fs.exists(path));
assertFalse("Path exists: " + path, fs.exists(path));
assertFalse("No deletion", fs.delete(path, true));
}

11
hadoop-project/pom.xml
View File

@ -306,6 +306,12 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-openstack</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@ -336,6 +342,11 @@
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.2.5</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>

153
hadoop-tools/hadoop-openstack/pom.xml Normal file
View File

@ -0,0 +1,153 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-openstack</artifactId>
<version>3.0.0-SNAPSHOT</version>
<name>Apache Hadoop OpenStack support</name>
<description>
This module contains code to support integration with OpenStack.
Currently this consists of a filesystem client to read data from
and write data to an OpenStack Swift object store.
</description>
<packaging>jar</packaging>
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<profiles>
<profile>
<id>tests-off</id>
<activation>
<file>
<missing>src/test/resources/auth-keys.xml</missing>
</file>
</activation>
<properties>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>tests-on</id>
<activation>
<file>
<exists>src/test/resources/auth-keys.xml</exists>
</file>
</activation>
<properties>
<maven.test.skip>false</maven.test.skip>
</properties>
</profile>
</profiles>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-module-markdown</artifactId>
<version>1.3</version>
</dependency>
</dependencies>
<configuration>
<inputEncoding>UTF-8</inputEncoding>
<outputEncoding>UTF-8</outputEncoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<configuration>
<dependencyDetailsEnabled>false</dependencyDetailsEnabled>
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>compile</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<scope>compile</scope>
</dependency>
<!-- Used for loading test resources and converting a File to byte[] -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>compile</scope>
</dependency>
<!-- Used for mocking dependencies -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.8.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

66
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
import org.codehaus.jackson.annotate.JsonProperty;
/**
* Class that represents authentication request to Openstack Keystone.
* Contains basic authentication information.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS
*/
public class ApiKeyAuthenticationRequest extends AuthenticationRequest {
/**
* Credentials for login
*/
private ApiKeyCredentials apiKeyCredentials;
/**
* API key auth
* @param tenantName tenant
* @param apiKeyCredentials credentials
*/
public ApiKeyAuthenticationRequest(String tenantName, ApiKeyCredentials apiKeyCredentials) {
this.tenantName = tenantName;
this.apiKeyCredentials = apiKeyCredentials;
}
/**
* @return credentials for login into Keystone
*/
@JsonProperty("RAX-KSKEY:apiKeyCredentials")
public ApiKeyCredentials getApiKeyCredentials() {
return apiKeyCredentials;
}
/**
* @param apiKeyCredentials credentials for login into Keystone
*/
public void setApiKeyCredentials(ApiKeyCredentials apiKeyCredentials) {
this.apiKeyCredentials = apiKeyCredentials;
}
@Override
public String toString() {
return "Auth as " +
"tenant '" + tenantName + "' "
+ apiKeyCredentials;
}
}

87
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyCredentials.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Describes credentials to log in Swift using Keystone authentication.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class ApiKeyCredentials {
/**
* user login
*/
private String username;
/**
* user password
*/
private String apikey;
/**
* default constructor
*/
public ApiKeyCredentials() {
}
/**
* @param username user login
* @param apikey user api key
*/
public ApiKeyCredentials(String username, String apikey) {
this.username = username;
this.apikey = apikey;
}
/**
* @return user api key
*/
public String getApiKey() {
return apikey;
}
/**
* @param apikey user api key
*/
public void setApiKey(String apikey) {
this.apikey = apikey;
}
/**
* @return login
*/
public String getUsername() {
return username;
}
/**
* @param username login
*/
public void setUsername(String username) {
this.username = username;
}
@Override
public String toString() {
return "user " +
"'" + username + '\'' +
" with key of length " + ((apikey == null) ? 0 : apikey.length());
}
}

57
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequest.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Class that represents authentication request to Openstack Keystone.
* Contains basic authentication information.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class AuthenticationRequest {
/**
* tenant name
*/
protected String tenantName;
public AuthenticationRequest() {
}
/**
* @return tenant name for Keystone authorization
*/
public String getTenantName() {
return tenantName;
}
/**
* @param tenantName tenant name for authorization
*/
public void setTenantName(String tenantName) {
this.tenantName = tenantName;
}
@Override
public String toString() {
return "AuthenticationRequest{" +
"tenantName='" + tenantName + '\'' +
'}';
}
}

59
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequestWrapper.java Normal file
View File

@ -0,0 +1,59 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* This class is used for correct hierarchy mapping of
* Keystone authentication model and java code.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class AuthenticationRequestWrapper {
/**
* authentication request
*/
private AuthenticationRequest auth;
/**
* default constructor used for json parsing
*/
public AuthenticationRequestWrapper() {
}
/**
* @param auth authentication requests
*/
public AuthenticationRequestWrapper(AuthenticationRequest auth) {
this.auth = auth;
}
/**
* @return authentication request
*/
public AuthenticationRequest getAuth() {
return auth;
}
/**
* @param auth authentication request
*/
public void setAuth(AuthenticationRequest auth) {
this.auth = auth;
}
}

69
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationResponse.java Normal file
View File

@ -0,0 +1,69 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
import org.apache.hadoop.fs.swift.auth.entities.AccessToken;
import org.apache.hadoop.fs.swift.auth.entities.Catalog;
import org.apache.hadoop.fs.swift.auth.entities.User;
import java.util.List;
/**
* Response from KeyStone deserialized into AuthenticationResponse class.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class AuthenticationResponse {
private Object metadata;
private List<Catalog> serviceCatalog;
private User user;
private AccessToken token;
public Object getMetadata() {
return metadata;
}
public void setMetadata(Object metadata) {
this.metadata = metadata;
}
public List<Catalog> getServiceCatalog() {
return serviceCatalog;
}
public void setServiceCatalog(List<Catalog> serviceCatalog) {
this.serviceCatalog = serviceCatalog;
}
public User getUser() {
return user;
}
public void setUser(User user) {
this.user = user;
}
public AccessToken getToken() {
return token;
}
public void setToken(AccessToken token) {
this.token = token;
}
}

47
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationWrapper.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* This class is used for correct hierarchy mapping of
* Keystone authentication model and java code
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class AuthenticationWrapper {
/**
* authentication response field
*/
private AuthenticationResponse access;
/**
* @return authentication response
*/
public AuthenticationResponse getAccess() {
return access;
}
/**
* @param access sets authentication response
*/
public void setAccess(AuthenticationResponse access) {
this.access = access;
}
}

59
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeyStoneAuthRequest.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Class that represents authentication to OpenStack Keystone.
* Contains basic authentication information.
* Used when {@link ApiKeyAuthenticationRequest} is not applicable.
* (problem with different Keystone installations/versions/modifications)
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class KeyStoneAuthRequest extends AuthenticationRequest {
/**
* Credentials for Keystone authentication
*/
private KeystoneApiKeyCredentials apiAccessKeyCredentials;
/**
* @param tenant Keystone tenant name for authentication
* @param apiAccessKeyCredentials Credentials for authentication
*/
public KeyStoneAuthRequest(String tenant, KeystoneApiKeyCredentials apiAccessKeyCredentials) {
this.apiAccessKeyCredentials = apiAccessKeyCredentials;
this.tenantName = tenant;
}
public KeystoneApiKeyCredentials getApiAccessKeyCredentials() {
return apiAccessKeyCredentials;
}
public void setApiAccessKeyCredentials(KeystoneApiKeyCredentials apiAccessKeyCredentials) {
this.apiAccessKeyCredentials = apiAccessKeyCredentials;
}
@Override
public String toString() {
return "KeyStoneAuthRequest as " +
"tenant '" + tenantName + "' "
+ apiAccessKeyCredentials;
}
}

66
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeystoneApiKeyCredentials.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Class for Keystone authentication.
* Used when {@link ApiKeyCredentials} is not applicable
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class KeystoneApiKeyCredentials {
/**
* User access key
*/
private String accessKey;
/**
* User access secret
*/
private String secretKey;
public KeystoneApiKeyCredentials(String accessKey, String secretKey) {
this.accessKey = accessKey;
this.secretKey = secretKey;
}
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
public String getSecretKey() {
return secretKey;
}
public void setSecretKey(String secretKey) {
this.secretKey = secretKey;
}
@Override
public String toString() {
return "user " +
"'" + accessKey + '\'' +
" with key of length " + ((secretKey == null) ? 0 : secretKey.length());
}
}

62
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordAuthenticationRequest.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Class that represents authentication request to Openstack Keystone.
* Contains basic authentication information.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class PasswordAuthenticationRequest extends AuthenticationRequest {
/**
* Credentials for login
*/
private PasswordCredentials passwordCredentials;
/**
* @param tenantName tenant
* @param passwordCredentials password credentials
*/
public PasswordAuthenticationRequest(String tenantName, PasswordCredentials passwordCredentials) {
this.tenantName = tenantName;
this.passwordCredentials = passwordCredentials;
}
/**
* @return credentials for login into Keystone
*/
public PasswordCredentials getPasswordCredentials() {
return passwordCredentials;
}
/**
* @param passwordCredentials credentials for login into Keystone
*/
public void setPasswordCredentials(PasswordCredentials passwordCredentials) {
this.passwordCredentials = passwordCredentials;
}
@Override
public String toString() {
return "Authenticate as " +
"tenant '" + tenantName + "' "
+ passwordCredentials;
}
}

87
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordCredentials.java Normal file
View File

@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Describes credentials to log in Swift using Keystone authentication.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class PasswordCredentials {
/**
* user login
*/
private String username;
/**
* user password
*/
private String password;
/**
* default constructor
*/
public PasswordCredentials() {
}
/**
* @param username user login
* @param password user password
*/
public PasswordCredentials(String username, String password) {
this.username = username;
this.password = password;
}
/**
* @return user password
*/
public String getPassword() {
return password;
}
/**
* @param password user password
*/
public void setPassword(String password) {
this.password = password;
}
/**
* @return login
*/
public String getUsername() {
return username;
}
/**
* @param username login
*/
public void setUsername(String username) {
this.username = username;
}
@Override
public String toString() {
return "user '" + username + '\'' +
" with password of length " + ((password == null) ? 0 : password.length());
}
}

97
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/Roles.java Normal file
View File

@ -0,0 +1,97 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth;
/**
* Describes user roles in Openstack system.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
public class Roles {
/**
* role name
*/
private String name;
/**
* This field user in RackSpace auth model
*/
private String id;
/**
* This field user in RackSpace auth model
*/
private String description;
/**
* Service id used in HP public Cloud
*/
private String serviceId;
/**
* Service id used in HP public Cloud
*/
private String tenantId;
/**
* @return role name
*/
public String getName() {
return name;
}
/**
* @param name role name
*/
public void setName(String name) {
this.name = name;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getServiceId() {
return serviceId;
}
public void setServiceId(String serviceId) {
this.serviceId = serviceId;
}
public String getTenantId() {
return tenantId;
}
public void setTenantId(String tenantId) {
this.tenantId = tenantId;
}
}

107
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java Normal file
View File

@ -0,0 +1,107 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth.entities;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
/**
* Access token representation of Openstack Keystone authentication.
* Class holds token id, tenant and expiration time.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*
* Example:
* <pre>
* "token" : {
* "RAX-AUTH:authenticatedBy" : [ "APIKEY" ],
* "expires" : "2013-07-12T05:19:24.685-05:00",
* "id" : "8bbea4215113abdab9d4c8fb0d37",
* "tenant" : { "id" : "01011970",
* "name" : "77777"
* }
* }
* </pre>
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class AccessToken {
/**
* token expiration time
*/
private String expires;
/**
* token id
*/
private String id;
/**
* tenant name for whom id is attached
*/
private Tenant tenant;
/**
* @return token expiration time
*/
public String getExpires() {
return expires;
}
/**
* @param expires the token expiration time
*/
public void setExpires(String expires) {
this.expires = expires;
}
/**
* @return token value
*/
public String getId() {
return id;
}
/**
* @param id token value
*/
public void setId(String id) {
this.id = id;
}
/**
* @return tenant authenticated in Openstack Keystone
*/
public Tenant getTenant() {
return tenant;
}
/**
* @param tenant tenant authenticated in Openstack Keystone
*/
public void setTenant(Tenant tenant) {
this.tenant = tenant;
}
@Override
public String toString() {
return "AccessToken{" +
"id='" + id + '\'' +
", tenant=" + tenant +
", expires='" + expires + '\'' +
'}';
}
}

107
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java Normal file
View File

@ -0,0 +1,107 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth.entities;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import java.util.List;
/**
* Describes Openstack Swift REST endpoints.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class Catalog {
/**
* List of valid swift endpoints
*/
private List<Endpoint> endpoints;
/**
* endpoint links are additional information description
* which aren't used in Hadoop and Swift integration scope
*/
private List<Object> endpoints_links;
/**
* Openstack REST service name. In our case name = "keystone"
*/
private String name;
/**
* Type of REST service. In our case type = "identity"
*/
private String type;
/**
* @return List of endpoints
*/
public List<Endpoint> getEndpoints() {
return endpoints;
}
/**
* @param endpoints list of endpoints
*/
public void setEndpoints(List<Endpoint> endpoints) {
this.endpoints = endpoints;
}
/**
* @return list of endpoint links
*/
public List<Object> getEndpoints_links() {
return endpoints_links;
}
/**
* @param endpoints_links list of endpoint links
*/
public void setEndpoints_links(List<Object> endpoints_links) {
this.endpoints_links = endpoints_links;
}
/**
* @return name of Openstack REST service
*/
public String getName() {
return name;
}
/**
* @param name of Openstack REST service
*/
public void setName(String name) {
this.name = name;
}
/**
* @return type of Openstack REST service
*/
public String getType() {
return type;
}
/**
* @param type of REST service
*/
public void setType(String type) {
this.type = type;
}
}

194
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java Normal file
View File

@ -0,0 +1,194 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth.entities;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import java.net.URI;
/**
* Openstack Swift endpoint description.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class Endpoint {
/**
* endpoint id
*/
private String id;
/**
* Keystone admin URL
*/
private URI adminURL;
/**
* Keystone internal URL
*/
private URI internalURL;
/**
* public accessible URL
*/
private URI publicURL;
/**
* public accessible URL#2
*/
private URI publicURL2;
/**
* Openstack region name
*/
private String region;
/**
* This field is used in RackSpace authentication model
*/
private String tenantId;
/**
* This field user in RackSpace auth model
*/
private String versionId;
/**
* This field user in RackSpace auth model
*/
private String versionInfo;
/**
* This field user in RackSpace auth model
*/
private String versionList;
/**
* @return endpoint id
*/
public String getId() {
return id;
}
/**
* @param id endpoint id
*/
public void setId(String id) {
this.id = id;
}
/**
* @return Keystone admin URL
*/
public URI getAdminURL() {
return adminURL;
}
/**
* @param adminURL Keystone admin URL
*/
public void setAdminURL(URI adminURL) {
this.adminURL = adminURL;
}
/**
* @return internal Keystone
*/
public URI getInternalURL() {
return internalURL;
}
/**
* @param internalURL Keystone internal URL
*/
public void setInternalURL(URI internalURL) {
this.internalURL = internalURL;
}
/**
* @return public accessible URL
*/
public URI getPublicURL() {
return publicURL;
}
/**
* @param publicURL public URL
*/
public void setPublicURL(URI publicURL) {
this.publicURL = publicURL;
}
public URI getPublicURL2() {
return publicURL2;
}
public void setPublicURL2(URI publicURL2) {
this.publicURL2 = publicURL2;
}
/**
* @return Openstack region name
*/
public String getRegion() {
return region;
}
/**
* @param region Openstack region name
*/
public void setRegion(String region) {
this.region = region;
}
public String getTenantId() {
return tenantId;
}
public void setTenantId(String tenantId) {
this.tenantId = tenantId;
}
public String getVersionId() {
return versionId;
}
public void setVersionId(String versionId) {
this.versionId = versionId;
}
public String getVersionInfo() {
return versionInfo;
}
public void setVersionInfo(String versionInfo) {
this.versionInfo = versionInfo;
}
public String getVersionList() {
return versionList;
}
public void setVersionList(String versionList) {
this.versionList = versionList;
}
}

107
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java Normal file
View File

@ -0,0 +1,107 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth.entities;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
/**
* Tenant is abstraction in Openstack which describes all account
* information and user privileges in system.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class Tenant {
/**
* tenant id
*/
private String id;
/**
* tenant short description which Keystone returns
*/
private String description;
/**
* boolean enabled user account or no
*/
private boolean enabled;
/**
* tenant human readable name
*/
private String name;
/**
* @return tenant name
*/
public String getName() {
return name;
}
/**
* @param name tenant name
*/
public void setName(String name) {
this.name = name;
}
/**
* @return true if account enabled and false otherwise
*/
public boolean isEnabled() {
return enabled;
}
/**
* @param enabled enable or disable
*/
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
/**
* @return account short description
*/
public String getDescription() {
return description;
}
/**
* @param description set account description
*/
public void setDescription(String description) {
this.description = description;
}
/**
* @return set tenant id
*/
public String getId() {
return id;
}
/**
* @param id tenant id
*/
public void setId(String id) {
this.id = id;
}
}

132
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java Normal file
View File

@ -0,0 +1,132 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.auth.entities;
import org.apache.hadoop.fs.swift.auth.Roles;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import java.util.List;
/**
* Describes user entity in Keystone
* In different Swift installations User is represented differently.
* To avoid any JSON deserialization failures this entity is ignored.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class User {
/**
* user id in Keystone
*/
private String id;
/**
* user human readable name
*/
private String name;
/**
* user roles in Keystone
*/
private List<Roles> roles;
/**
* links to user roles
*/
private List<Object> roles_links;
/**
* human readable username in Keystone
*/
private String username;
/**
* @return user id
*/
public String getId() {
return id;
}
/**
* @param id user id
*/
public void setId(String id) {
this.id = id;
}
/**
* @return user name
*/
public String getName() {
return name;
}
/**
* @param name user name
*/
public void setName(String name) {
this.name = name;
}
/**
* @return user roles
*/
public List<Roles> getRoles() {
return roles;
}
/**
* @param roles sets user roles
*/
public void setRoles(List<Roles> roles) {
this.roles = roles;
}
/**
* @return user roles links
*/
public List<Object> getRoles_links() {
return roles_links;
}
/**
* @param roles_links user roles links
*/
public void setRoles_links(List<Object> roles_links) {
this.roles_links = roles_links;
}
/**
* @return username
*/
public String getUsername() {
return username;
}
/**
* @param username human readable user name
*/
public void setUsername(String username) {
this.username = username;
}
}

48
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
import org.apache.commons.httpclient.HttpMethod;
import java.net.URI;
/**
* An exception raised when an authentication request was rejected
*/
public class SwiftAuthenticationFailedException extends SwiftInvalidResponseException {
public SwiftAuthenticationFailedException(String message,
int statusCode,
String operation,
URI uri) {
super(message, statusCode, operation, uri);
}
public SwiftAuthenticationFailedException(String message,
String operation,
URI uri,
HttpMethod method) {
super(message, operation, uri, method);
}
@Override
public String exceptionTitle() {
return "Authentication Failure";
}
}

49
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
import org.apache.commons.httpclient.HttpMethod;
import java.net.URI;
/**
* Thrown to indicate that data locality can't be calculated or requested path is incorrect.
* Data locality can't be calculated if Openstack Swift version is old.
*/
public class SwiftBadRequestException extends SwiftInvalidResponseException {
public SwiftBadRequestException(String message,
String operation,
URI uri,
HttpMethod method) {
super(message, operation, uri, method);
}
public SwiftBadRequestException(String message,
int statusCode,
String operation,
URI uri) {
super(message, statusCode, operation, uri);
}
@Override
public String exceptionTitle() {
return "BadRequest";
}
}

33
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConfigurationException.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Exception raised to indicate there is some problem with how the Swift FS
* is configured
*/
public class SwiftConfigurationException extends SwiftException {
public SwiftConfigurationException(String message) {
super(message);
}
public SwiftConfigurationException(String message, Throwable cause) {
super(message, cause);
}
}

36
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionClosedException.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Exception raised when an attempt is made to use a closed stream
*/
public class SwiftConnectionClosedException extends SwiftException {
public static final String MESSAGE =
"Connection to Swift service has been closed";
public SwiftConnectionClosedException() {
super(MESSAGE);
}
public SwiftConnectionClosedException(String reason) {
super(MESSAGE + ": " + reason);
}
}

35
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionException.java Normal file
View File

@ -0,0 +1,35 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Thrown to indicate that connection is lost or failed to be made
*/
public class SwiftConnectionException extends SwiftException {
public SwiftConnectionException() {
}
public SwiftConnectionException(String message) {
super(message);
}
public SwiftConnectionException(String message, Throwable cause) {
super(message, cause);
}
}

43
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftException.java Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
import java.io.IOException;
/**
* A Swift-specific exception -subclasses exist
* for various specific problems.
*/
public class SwiftException extends IOException {
public SwiftException() {
super();
}
public SwiftException(String message) {
super(message);
}
public SwiftException(String message, Throwable cause) {
super(message, cause);
}
public SwiftException(Throwable cause) {
super(cause);
}
}

38
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInternalStateException.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* The internal state of the Swift client is wrong -presumably a sign
* of some bug
*/
public class SwiftInternalStateException extends SwiftException {
public SwiftInternalStateException(String message) {
super(message);
}
public SwiftInternalStateException(String message, Throwable cause) {
super(message, cause);
}
public SwiftInternalStateException(Throwable cause) {
super(cause);
}
}

117
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
import org.apache.commons.httpclient.HttpMethod;
import java.io.IOException;
import java.net.URI;
/**
* Exception raised when the HTTP code is invalid. The status code,
* method name and operation URI are all in the response.
*/
public class SwiftInvalidResponseException extends SwiftConnectionException {
public final int statusCode;
public final String operation;
public final URI uri;
public final String body;
public SwiftInvalidResponseException(String message,
int statusCode,
String operation,
URI uri) {
super(message);
this.statusCode = statusCode;
this.operation = operation;
this.uri = uri;
this.body = "";
}
public SwiftInvalidResponseException(String message,
String operation,
URI uri,
HttpMethod method) {
super(message);
this.statusCode = method.getStatusCode();
this.operation = operation;
this.uri = uri;
String bodyAsString;
try {
bodyAsString = method.getResponseBodyAsString();
if (bodyAsString == null) {
bodyAsString = "";
}
} catch (IOException e) {
bodyAsString = "";
}
this.body = bodyAsString;
}
public int getStatusCode() {
return statusCode;
}
public String getOperation() {
return operation;
}
public URI getUri() {
return uri;
}
public String getBody() {
return body;
}
/**
* Override point: title of an exception -this is used in the
* toString() method.
* @return the new exception title
*/
public String exceptionTitle() {
return "Invalid Response";
}
/**
* Build a description that includes the exception title, the URI,
* the message, the status code -and any body of the response
* @return the string value for display
*/
@Override
public String toString() {
StringBuilder msg = new StringBuilder();
msg.append(exceptionTitle());
msg.append(": ");
msg.append(getMessage());
msg.append(" ");
msg.append(operation);
msg.append(" ");
msg.append(uri);
msg.append(" => ");
msg.append(statusCode);
if (body != null && !body.isEmpty()) {
msg.append(" : ");
msg.append(body);
}
return msg.toString();
}
}

33
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Exception raised when the J/O mapping fails.
*/
public class SwiftJsonMarshallingException extends SwiftException {
public SwiftJsonMarshallingException(String message) {
super(message);
}
public SwiftJsonMarshallingException(String message, Throwable cause) {
super(message, cause);
}
}

43
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
import org.apache.hadoop.fs.Path;
/**
* Exception raised when an operation is meant to work on a directory, but
* the target path is not a directory
*/
public class SwiftNotDirectoryException extends SwiftException {
private final Path path;
public SwiftNotDirectoryException(Path path) {
this(path, "");
}
public SwiftNotDirectoryException(Path path,
String message) {
super(path.toString() + message);
this.path = path;
}
public Path getPath() {
return path;
}
}

35
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Used to relay exceptions upstream from the inner implementation
* to the public API, where it is downgraded to a log+failure.
* Making it visible internally aids testing
*/
public class SwiftOperationFailedException extends SwiftException {
public SwiftOperationFailedException(String message) {
super(message);
}
public SwiftOperationFailedException(String message, Throwable cause) {
super(message, cause);
}
}

33
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Exception raised when trying to create a file that already exists
* and the overwrite flag is set to false.
*/
public class SwiftPathExistsException extends SwiftException {
public SwiftPathExistsException(String message) {
super(message);
}
public SwiftPathExistsException(String message, Throwable cause) {
super(message, cause);
}
}

37
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
import org.apache.commons.httpclient.HttpMethod;
import java.net.URI;
/**
* Exception raised if a Swift endpoint returned a HTTP response indicating
* the caller is being throttled.
*/
public class SwiftThrottledRequestException extends
SwiftInvalidResponseException {
public SwiftThrottledRequestException(String message,
String operation,
URI uri,
HttpMethod method) {
super(message, operation, uri, method);
}
}

30
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.exceptions;
/**
* Exception raised on an unsupported feature in the FS API -such as
* <code>append()</code>
*/
public class SwiftUnsupportedFeatureException extends SwiftException {
public SwiftUnsupportedFeatureException(String message) {
super(message);
}
}

41
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
/**
* Implementation for SwiftRestClient to make copy requests.
* COPY is a method that came with WebDAV (RFC2518), and is not something that
* can be handled by all proxies en-route to a filesystem.
*/
class CopyMethod extends EntityEnclosingMethod {
public CopyMethod(String uri) {
super(uri);
}
/**
* @return http method name
*/
@Override
public String getName() {
return "COPY";
}
}

99
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
/**
* Variant of Hadoop Netutils exception wrapping with URI awareness and
* available in branch-1 too.
*/
public class ExceptionDiags {
private static final Log LOG = LogFactory.getLog(ExceptionDiags.class);
/** text to point users elsewhere: {@value} */
private static final String FOR_MORE_DETAILS_SEE
= " For more details see: ";
/** text included in wrapped exceptions if the host is null: {@value} */
public static final String UNKNOWN_HOST = "(unknown)";
/** Base URL of the Hadoop Wiki: {@value} */
public static final String HADOOP_WIKI = "http://wiki.apache.org/hadoop/";
/**
* Take an IOException and a URI, wrap it where possible with
* something that includes the URI
*
* @param dest target URI
* @param operation operation
* @param exception the caught exception.
* @return an exception to throw
*/
public static IOException wrapException(final String dest,
final String operation,
final IOException exception) {
String action = operation + " " + dest;
String xref = null;
if (exception instanceof ConnectException) {
xref = "ConnectionRefused";
} else if (exception instanceof UnknownHostException) {
xref = "UnknownHost";
} else if (exception instanceof SocketTimeoutException) {
xref = "SocketTimeout";
} else if (exception instanceof NoRouteToHostException) {
xref = "NoRouteToHost";
}
String msg = action
+ " failed on exception: "
+ exception;
if (xref != null) {
msg = msg + ";" + see(xref);
}
return wrapWithMessage(exception, msg);
}
private static String see(final String entry) {
return FOR_MORE_DETAILS_SEE + HADOOP_WIKI + entry;
}
@SuppressWarnings("unchecked")
private static <T extends IOException> T wrapWithMessage(
T exception, String msg) {
Class<? extends Throwable> clazz = exception.getClass();
try {
Constructor<? extends Throwable> ctor =
clazz.getConstructor(String.class);
Throwable t = ctor.newInstance(msg);
return (T) (t.initCause(exception));
} catch (Throwable e) {
LOG.warn("Unable to wrap exception of type " +
clazz + ": it has no (String) constructor", e);
return exception;
}
}
}

45
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
/**
* Response tuple from GET operations; combines the input stream with the content length
*/
public class HttpBodyContent {
private final long contentLength;
private final HttpInputStreamWithRelease inputStream;
/**
* build a body response
* @param inputStream input stream from the operatin
* @param contentLength length of content; may be -1 for "don't know"
*/
public HttpBodyContent(HttpInputStreamWithRelease inputStream,
long contentLength) {
this.contentLength = contentLength;
this.inputStream = inputStream;
}
public long getContentLength() {
return contentLength;
}
public HttpInputStreamWithRelease getInputStream() {
return inputStream;
}
}

235
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java Normal file
View File

@ -0,0 +1,235 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
/**
* This replaces the input stream release class from JetS3t and AWS;
* # Failures in the constructor are relayed up instead of simply logged.
* # it is set up to be more robust at teardown
* # release logic is thread safe
* Note that the thread safety of the inner stream contains no thread
* safety guarantees -this stream is not to be read across streams.
* The thread safety logic here is to ensure that even if somebody ignores
* that rule, the release code does not get entered twice -and that
* any release in one thread is picked up by read operations in all others.
*/
public class HttpInputStreamWithRelease extends InputStream {
private static final Log LOG =
LogFactory.getLog(HttpInputStreamWithRelease.class);
private final URI uri;
private HttpMethod method;
//flag to say the stream is released -volatile so that read operations
//pick it up even while unsynchronized.
private volatile boolean released;
//volatile flag to verify that data is consumed.
private volatile boolean dataConsumed;
private InputStream inStream;
/**
* In debug builds, this is filled in with the construction-time
* stack, which is then included in logs from the finalize(), method.
*/
private final Exception constructionStack;
/**
* Why the stream is closed
*/
private String reasonClosed = "unopened";
public HttpInputStreamWithRelease(URI uri, HttpMethod method) throws
IOException {
this.uri = uri;
this.method = method;
constructionStack = LOG.isDebugEnabled() ? new Exception("stack") : null;
if (method == null) {
throw new IllegalArgumentException("Null 'method' parameter ");
}
try {
inStream = method.getResponseBodyAsStream();
} catch (IOException e) {
inStream = new ByteArrayInputStream(new byte[]{});
throw releaseAndRethrow("getResponseBodyAsStream() in constructor -" + e, e);
}
}
@Override
public void close() throws IOException {
release("close()", null);
}
/**
* Release logic
* @param reason reason for release (used in debug messages)
* @param ex exception that is a cause -null for non-exceptional releases
* @return true if the release took place here
* @throws IOException if the abort or close operations failed.
*/
private synchronized boolean release(String reason, Exception ex) throws
IOException {
if (!released) {
reasonClosed = reason;
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Releasing connection to " + uri + ": " + reason, ex);
}
if (method != null) {
if (!dataConsumed) {
method.abort();
}
method.releaseConnection();
}
if (inStream != null) {
//this guard may seem un-needed, but a stack trace seen
//on the JetS3t predecessor implied that it
//is useful
inStream.close();
}
return true;
} finally {
//if something went wrong here, we do not want the release() operation
//to try and do anything in advance.
released = true;
dataConsumed = true;
}
} else {
return false;
}
}
/**
* Release the method, using the exception as a cause
* @param operation operation that failed
* @param ex the exception which triggered it.
* @return the exception to throw
*/
private IOException releaseAndRethrow(String operation, IOException ex) {
try {
release(operation, ex);
} catch (IOException ioe) {
LOG.debug("Exception during release: " + operation + " - " + ioe, ioe);
//make this the exception if there was none before
if (ex == null) {
ex = ioe;
}
}
return ex;
}
/**
* Assume that the connection is not released: throws an exception if it is
* @throws SwiftConnectionClosedException
*/
private synchronized void assumeNotReleased() throws SwiftConnectionClosedException {
if (released || inStream == null) {
throw new SwiftConnectionClosedException(reasonClosed);
}
}
@Override
public int available() throws IOException {
assumeNotReleased();
try {
return inStream.available();
} catch (IOException e) {
throw releaseAndRethrow("available() failed -" + e, e);
}
}
@Override
public int read() throws IOException {
assumeNotReleased();
int read = 0;
try {
read = inStream.read();
} catch (EOFException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("EOF exception " + e, e);
}
read = -1;
} catch (IOException e) {
throw releaseAndRethrow("read()", e);
}
if (read < 0) {
dataConsumed = true;
release("read() -all data consumed", null);
}
return read;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
SwiftUtils.validateReadArgs(b, off, len);
//if the stream is already closed, then report an exception.
assumeNotReleased();
//now read in a buffer, reacting differently to different operations
int read;
try {
read = inStream.read(b, off, len);
} catch (EOFException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("EOF exception " + e, e);
}
read = -1;
} catch (IOException e) {
throw releaseAndRethrow("read(b, off, " + len + ")", e);
}
if (read < 0) {
dataConsumed = true;
release("read() -all data consumed", null);
}
return read;
}
/**
* Finalizer does release the stream, but also logs at WARN level
* including the URI at fault
*/
@Override
protected void finalize() {
try {
if (release("finalize()", constructionStack)) {
LOG.warn("input stream of " + uri
+ " not closed properly -cleaned up in finalize()");
}
} catch (Exception e) {
//swallow anything that failed here
LOG.warn("Exception while releasing " + uri + "in finalizer",
e);
}
}
@Override
public String toString() {
return "HttpInputStreamWithRelease working with " + uri
+" released=" + released
+" dataConsumed=" + dataConsumed;
}
}

224
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java Normal file
View File

@ -0,0 +1,224 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import java.net.URI;
import java.util.Properties;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*;
/**
* This class implements the binding logic between Hadoop configurations
* and the swift rest client.
* <p/>
* The swift rest client takes a Properties instance containing
* the string values it uses to bind to a swift endpoint.
* <p/>
* This class extracts the values for a specific filesystem endpoint
* and then builds an appropriate Properties file.
*/
public final class RestClientBindings {
private static final Log LOG = LogFactory.getLog(RestClientBindings.class);
public static final String E_INVALID_NAME = "Invalid swift hostname '%s':" +
" hostname must in form container.service";
/**
* Public for testing : build the full prefix for use in resolving
* configuration items
*
* @param service service to use
* @return the prefix string <i>without any trailing "."</i>
*/
public static String buildSwiftInstancePrefix(String service) {
return SWIFT_SERVICE_PREFIX + service;
}
/**
* Raise an exception for an invalid service name
*
* @param hostname hostname that was being parsed
* @return an exception to throw
*/
private static SwiftConfigurationException invalidName(String hostname) {
return new SwiftConfigurationException(
String.format(E_INVALID_NAME, hostname));
}
/**
* Get the container name from the hostname -the single element before the
* first "." in the hostname
*
* @param hostname hostname to split
* @return the container
* @throws SwiftConfigurationException
*/
public static String extractContainerName(String hostname) throws
SwiftConfigurationException {
int i = hostname.indexOf(".");
if (i <= 0) {
throw invalidName(hostname);
}
return hostname.substring(0, i);
}
public static String extractContainerName(URI uri) throws
SwiftConfigurationException {
return extractContainerName(uri.getHost());
}
/**
* Get the service name from a longer hostname string
*
* @param hostname hostname
* @return the separated out service name
* @throws SwiftConfigurationException if the hostname was invalid
*/
public static String extractServiceName(String hostname) throws
SwiftConfigurationException {
int i = hostname.indexOf(".");
if (i <= 0) {
throw invalidName(hostname);
}
String service = hostname.substring(i + 1);
if (service.isEmpty() || service.contains(".")) {
//empty service contains dots in -not currently supported
throw invalidName(hostname);
}
return service;
}
public static String extractServiceName(URI uri) throws
SwiftConfigurationException {
return extractServiceName(uri.getHost());
}
/**
* Build a properties instance bound to the configuration file -using
* the filesystem URI as the source of the information.
*
* @param fsURI filesystem URI
* @param conf configuration
* @return a properties file with the instance-specific properties extracted
* and bound to the swift client properties.
* @throws SwiftConfigurationException if the configuration is invalid
*/
public static Properties bind(URI fsURI, Configuration conf) throws
SwiftConfigurationException {
String host = fsURI.getHost();
if (host == null || host.isEmpty()) {
//expect shortnames -> conf names
throw invalidName(host);
}
String container = extractContainerName(host);
String service = extractServiceName(host);
//build filename schema
String prefix = buildSwiftInstancePrefix(service);
if (LOG.isDebugEnabled()) {
LOG.debug("Filesystem " + fsURI
+ " is using configuration keys " + prefix);
}
Properties props = new Properties();
props.setProperty(SWIFT_SERVICE_PROPERTY, service);
props.setProperty(SWIFT_CONTAINER_PROPERTY, container);
copy(conf, prefix + DOT_AUTH_URL, props, SWIFT_AUTH_PROPERTY, true);
copy(conf, prefix + DOT_USERNAME, props, SWIFT_USERNAME_PROPERTY, true);
copy(conf, prefix + DOT_APIKEY, props, SWIFT_APIKEY_PROPERTY, false);
copy(conf, prefix + DOT_PASSWORD, props, SWIFT_PASSWORD_PROPERTY,
props.contains(SWIFT_APIKEY_PROPERTY) ? true : false);
copy(conf, prefix + DOT_TENANT, props, SWIFT_TENANT_PROPERTY, false);
copy(conf, prefix + DOT_REGION, props, SWIFT_REGION_PROPERTY, false);
copy(conf, prefix + DOT_HTTP_PORT, props, SWIFT_HTTP_PORT_PROPERTY, false);
copy(conf, prefix +
DOT_HTTPS_PORT, props, SWIFT_HTTPS_PORT_PROPERTY, false);
copyBool(conf, prefix + DOT_PUBLIC, props, SWIFT_PUBLIC_PROPERTY, false);
copyBool(conf, prefix + DOT_LOCATION_AWARE, props,
SWIFT_LOCATION_AWARE_PROPERTY, false);
return props;
}
/**
* Extract a boolean value from the configuration and copy it to the
* properties instance.
* @param conf source configuration
* @param confKey key in the configuration file
* @param props destination property set
* @param propsKey key in the property set
* @param defVal default value
*/
private static void copyBool(Configuration conf,
String confKey,
Properties props,
String propsKey,
boolean defVal) {
boolean b = conf.getBoolean(confKey, defVal);
props.setProperty(propsKey, Boolean.toString(b));
}
private static void set(Properties props, String key, String optVal) {
if (optVal != null) {
props.setProperty(key, optVal);
}
}
/**
* Copy a (trimmed) property from the configuration file to the properties file.
* <p/>
* If marked as required and not found in the configuration, an
* exception is raised.
* If not required -and missing- then the property will not be set.
* In this case, if the property is already in the Properties instance,
* it will remain untouched.
*
* @param conf source configuration
* @param confKey key in the configuration file
* @param props destination property set
* @param propsKey key in the property set
* @param required is the property required
* @throws SwiftConfigurationException if the property is required but was
* not found in the configuration instance.
*/
public static void copy(Configuration conf, String confKey, Properties props,
String propsKey,
boolean required) throws SwiftConfigurationException {
//TODO: replace. version compatibility issue conf.getTrimmed fails with NoSuchMethodError
String val = conf.get(confKey);
if (val != null) {
val = val.trim();
}
if (required && val == null) {
throw new SwiftConfigurationException(
"Missing mandatory configuration option: "
+
confKey);
}
set(props, propsKey, val);
}
}

270
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java Normal file
View File

@ -0,0 +1,270 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.hadoop.util.VersionInfo;
/**
* Constants used in the Swift REST protocol,
* and in the properties used to configure the {@link SwiftRestClient}.
*/
public class SwiftProtocolConstants {
/**
* Swift-specific header for authentication: {@value}
*/
public static final String HEADER_AUTH_KEY = "X-Auth-Token";
/**
* Default port used by Swift for HTTP
*/
public static final int SWIFT_HTTP_PORT = 8080;
/**
* Default port used by Swift Auth for HTTPS
*/
public static final int SWIFT_HTTPS_PORT = 443;
/** HTTP standard {@value} header */
public static final String HEADER_RANGE = "Range";
/** HTTP standard {@value} header */
public static final String HEADER_DESTINATION = "Destination";
/** HTTP standard {@value} header */
public static final String HEADER_LAST_MODIFIED = "Last-Modified";
/** HTTP standard {@value} header */
public static final String HEADER_CONTENT_LENGTH = "Content-Length";
/** HTTP standard {@value} header */
public static final String HEADER_CONTENT_RANGE = "Content-Range";
/**
* Patten for range headers
*/
public static final String SWIFT_RANGE_HEADER_FORMAT_PATTERN = "bytes=%d-%d";
/**
* section in the JSON catalog provided after auth listing the swift FS:
* {@value}
*/
public static final String SERVICE_CATALOG_SWIFT = "swift";
/**
* section in the JSON catalog provided after auth listing the cloudfiles;
* this is an alternate catalog entry name
* {@value}
*/
public static final String SERVICE_CATALOG_CLOUD_FILES = "cloudFiles";
/**
* section in the JSON catalog provided after auth listing the object store;
* this is an alternate catalog entry name
* {@value}
*/
public static final String SERVICE_CATALOG_OBJECT_STORE = "object-store";
/**
* entry in the swift catalog defining the prefix used to talk to objects
* {@value}
*/
public static final String SWIFT_OBJECT_AUTH_ENDPOINT =
"/object_endpoint/";
/**
* Swift-specific header: object manifest used in the final upload
* of a multipart operation: {@value}
*/
public static final String X_OBJECT_MANIFEST = "X-Object-Manifest";
/**
* Swift-specific header -#of objects in a container: {@value}
*/
public static final String X_CONTAINER_OBJECT_COUNT =
"X-Container-Object-Count";
/**
* Swift-specific header: no. of bytes used in a container {@value}
*/
public static final String X_CONTAINER_BYTES_USED = "X-Container-Bytes-Used";
/**
* Header to set when requesting the latest version of a file: : {@value}
*/
public static final String X_NEWEST = "X-Newest";
/**
* throttled response sent by some endpoints.
*/
public static final int SC_THROTTLED_498 = 498;
/**
* W3C recommended status code for throttled operations
*/
public static final int SC_TOO_MANY_REQUESTS_429 = 429;
public static final String FS_SWIFT = "fs.swift";
/**
* Prefix for all instance-specific values in the configuration: {@value}
*/
public static final String SWIFT_SERVICE_PREFIX = FS_SWIFT + ".service.";
/**
* timeout for all connections: {@value}
*/
public static final String SWIFT_CONNECTION_TIMEOUT =
FS_SWIFT + ".connect.timeout";
/**
* timeout for all connections: {@value}
*/
public static final String SWIFT_SOCKET_TIMEOUT =
FS_SWIFT + ".socket.timeout";
/**
* the default socket timeout in millis {@value}.
* This controls how long the connection waits for responses from
* servers.
*/
public static final int DEFAULT_SOCKET_TIMEOUT = 60000;
/**
* connection retry count for all connections: {@value}
*/
public static final String SWIFT_RETRY_COUNT =
FS_SWIFT + ".connect.retry.count";
/**
* delay in millis between bulk (delete, rename, copy operations: {@value}
*/
public static final String SWIFT_THROTTLE_DELAY =
FS_SWIFT + ".connect.throttle.delay";
/**
* the default throttle delay in millis {@value}
*/
public static final int DEFAULT_THROTTLE_DELAY = 0;
/**
* blocksize for all filesystems: {@value}
*/
public static final String SWIFT_BLOCKSIZE =
FS_SWIFT + ".blocksize";
/**
* the default blocksize for filesystems in KB: {@value}
*/
public static final int DEFAULT_SWIFT_BLOCKSIZE = 32 * 1024;
/**
* partition size for all filesystems in KB: {@value}
*/
public static final String SWIFT_PARTITION_SIZE =
FS_SWIFT + ".partsize";
/**
* The default partition size for uploads: {@value}
*/
public static final int DEFAULT_SWIFT_PARTITION_SIZE = 4608*1024;
/**
* request size for reads in KB: {@value}
*/
public static final String SWIFT_REQUEST_SIZE =
FS_SWIFT + ".requestsize";
/**
* The default reqeuest size for reads: {@value}
*/
public static final int DEFAULT_SWIFT_REQUEST_SIZE = 64;
public static final String HEADER_USER_AGENT="User-Agent";
/**
* The user agent sent in requests.
*/
public static final String SWIFT_USER_AGENT= "Apache Hadoop Swift Client "
+ VersionInfo.getBuildVersion();
/**
* Key for passing the service name as a property -not read from the
* configuration : {@value}
*/
public static final String DOT_SERVICE = ".SERVICE-NAME";
/**
* Key for passing the container name as a property -not read from the
* configuration : {@value}
*/
public static final String DOT_CONTAINER = ".CONTAINER-NAME";
public static final String DOT_AUTH_URL = ".auth.url";
public static final String DOT_TENANT = ".tenant";
public static final String DOT_USERNAME = ".username";
public static final String DOT_PASSWORD = ".password";
public static final String DOT_HTTP_PORT = ".http.port";
public static final String DOT_HTTPS_PORT = ".https.port";
public static final String DOT_REGION = ".region";
public static final String DOT_PROXY_HOST = ".proxy.host";
public static final String DOT_PROXY_PORT = ".proxy.port";
public static final String DOT_LOCATION_AWARE = ".location-aware";
public static final String DOT_APIKEY = ".apikey";
public static final String DOT_USE_APIKEY = ".useApikey";
/**
* flag to say use public URL
*/
public static final String DOT_PUBLIC = ".public";
public static final String SWIFT_SERVICE_PROPERTY = FS_SWIFT + DOT_SERVICE;
public static final String SWIFT_CONTAINER_PROPERTY = FS_SWIFT + DOT_CONTAINER;
public static final String SWIFT_AUTH_PROPERTY = FS_SWIFT + DOT_AUTH_URL;
public static final String SWIFT_TENANT_PROPERTY = FS_SWIFT + DOT_TENANT;
public static final String SWIFT_USERNAME_PROPERTY = FS_SWIFT + DOT_USERNAME;
public static final String SWIFT_PASSWORD_PROPERTY = FS_SWIFT + DOT_PASSWORD;
public static final String SWIFT_APIKEY_PROPERTY = FS_SWIFT + DOT_APIKEY;
public static final String SWIFT_HTTP_PORT_PROPERTY = FS_SWIFT + DOT_HTTP_PORT;
public static final String SWIFT_HTTPS_PORT_PROPERTY = FS_SWIFT
+ DOT_HTTPS_PORT;
public static final String SWIFT_REGION_PROPERTY = FS_SWIFT + DOT_REGION;
public static final String SWIFT_PUBLIC_PROPERTY = FS_SWIFT + DOT_PUBLIC;
public static final String SWIFT_USE_API_KEY_PROPERTY = FS_SWIFT + DOT_USE_APIKEY;
public static final String SWIFT_LOCATION_AWARE_PROPERTY = FS_SWIFT +
DOT_LOCATION_AWARE;
public static final String SWIFT_PROXY_HOST_PROPERTY = FS_SWIFT + DOT_PROXY_HOST;
public static final String SWIFT_PROXY_PORT_PROPERTY = FS_SWIFT + DOT_PROXY_PORT;
public static final String HTTP_ROUTE_DEFAULT_PROXY =
"http.route.default-proxy";
/**
* Topology to return when a block location is requested
*/
public static final String TOPOLOGY_PATH = "/swift/unknown";
/**
* Block location to return when a block location is requested
*/
public static final String BLOCK_LOCATION = "/default-rack/swift";
/**
* Default number of attempts to retry a connect request: {@value}
*/
static final int DEFAULT_RETRY_COUNT = 3;
/**
* Default timeout in milliseconds for connection requests: {@value}
*/
static final int DEFAULT_CONNECT_TIMEOUT = 15000;
}

1877
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java Normal file
View File

File diff suppressed because it is too large Load Diff

81
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html Normal file
View File

@ -0,0 +1,81 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<html>
<head>
<title>Swift Filesystem Client for Apache Hadoop</title>
</head>
<body>
<h1>
Swift Filesystem Client for Apache Hadoop
</h1>
<h2>Introduction</h2>
<div>This package provides support in Apache Hadoop for the OpenStack Swift
Key-Value store, allowing client applications -including MR Jobs- to
read and write data in Swift.
</div>
<div>Design Goals</div>
<ol>
<li>Give clients access to SwiftFS files, similar to S3n:</li>
<li>maybe: support a Swift Block store -- at least until Swift's
support for &gt;5GB files has stabilized.
</li>
<li>Support for data-locality if the Swift FS provides file location information</li>
<li>Support access to multiple Swift filesystems in the same client/task.</li>
<li>Authenticate using the Keystone APIs.</li>
<li>Avoid dependency on unmaintained libraries.</li>
</ol>
<h2>Supporting multiple Swift Filesystems</h2>
The goal of supporting multiple swift filesystems simultaneously changes how
clusters are named and authenticated. In Hadoop's S3 and S3N filesystems, the "bucket" into
which objects are stored is directly named in the URL, such as
<code>s3n://bucket/object1</code>. The Hadoop configuration contains a
single set of login credentials for S3 (username and key), which are used to
authenticate the HTTP operations.
For swift, we need to know not only which "container" name, but which credentials
to use to authenticate with it -and which URL to use for authentication.
This has led to a different design pattern from S3, as instead of simple bucket names,
the hostname of an S3 container is two-level, the name of the service provider
being the second path: <code>swift://bucket.service/</code>
The <code>service</code> portion of this domainame is used as a reference into
the client settings -and so identify the service provider of that container.
<h2>Testing</h2>
<div>
The client code can be tested against public or private Swift instances; the
public services are (at the time of writing -January 2013-), Rackspace and
HP Cloud. Testing against both instances is how interoperability
can be verified.
</div>
</body>
</html>

47
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import org.apache.hadoop.fs.BufferedFSInputStream;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
import java.io.IOException;
/**
* Add stricter compliance with the evolving FS specifications
*/
public class StrictBufferedFSInputStream extends BufferedFSInputStream {
public StrictBufferedFSInputStream(FSInputStream in,
int size) {
super(in, size);
}
@Override
public void seek(long pos) throws IOException {
if (pos < 0) {
throw new IOException("Negative position");
}
if (in == null) {
throw new SwiftConnectionClosedException("Stream closed");
}
super.seek(pos);
}
}

110
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
/**
* A subclass of {@link FileStatus} that contains the
* Swift-specific rules of when a file is considered to be a directory.
*/
public class SwiftFileStatus extends FileStatus {
public SwiftFileStatus() {
}
public SwiftFileStatus(long length,
boolean isdir,
int block_replication,
long blocksize, long modification_time, Path path) {
super(length, isdir, block_replication, blocksize, modification_time, path);
}
public SwiftFileStatus(long length,
boolean isdir,
int block_replication,
long blocksize,
long modification_time,
long access_time,
FsPermission permission,
String owner, String group, Path path) {
super(length, isdir, block_replication, blocksize, modification_time,
access_time, permission, owner, group, path);
}
//HDFS2+ only
public SwiftFileStatus(long length,
boolean isdir,
int block_replication,
long blocksize,
long modification_time,
long access_time,
FsPermission permission,
String owner, String group, Path symlink, Path path) {
super(length, isdir, block_replication, blocksize, modification_time,
access_time, permission, owner, group, symlink, path);
}
/**
* Declare that the path represents a directory, which in the
* SwiftNativeFileSystem means "is a directory or a 0 byte file"
*
* @return true if the status is considered to be a file
*/
@Override
public boolean isDir() {
return super.isDirectory() || getLen() == 0;
}
/**
* A entry is a file if it is not a directory.
* By implementing it <i>and not marking as an override</i> this
* subclass builds and runs in both Hadoop versions.
* @return the opposite value to {@link #isDir()}
*/
@Override
public boolean isFile() {
return !isDir();
}
/**
* Directory test
* @return true if the file is considered to be a directory
*/
public boolean isDirectory() {
return isDir();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(getClass().getSimpleName());
sb.append("{ ");
sb.append("path=").append(getPath());
sb.append("; isDirectory=").append(isDir());
sb.append("; length=").append(getLen());
sb.append("; blocksize=").append(getBlockSize());
sb.append("; modification_time=").append(getModificationTime());
sb.append("}");
return sb.toString();
}
}

714
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java Normal file
View File

@ -0,0 +1,714 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
import org.apache.hadoop.fs.swift.exceptions.SwiftPathExistsException;
import org.apache.hadoop.fs.swift.exceptions.SwiftUnsupportedFeatureException;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import org.apache.hadoop.util.Progressable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
/**
* Swift file system implementation. Extends Hadoop FileSystem
*/
public class SwiftNativeFileSystem extends FileSystem {
/** filesystem prefix: {@value} */
public static final String SWIFT = "swift";
private static final Log LOG =
LogFactory.getLog(SwiftNativeFileSystem.class);
/**
* path to user work directory for storing temporary files
*/
private Path workingDir;
/**
* Swift URI
*/
private URI uri;
/**
* reference to swiftFileSystemStore
*/
private SwiftNativeFileSystemStore store;
/**
* Default constructor for Hadoop
*/
public SwiftNativeFileSystem() {
// set client in initialize()
}
/**
* This constructor used for testing purposes
*/
public SwiftNativeFileSystem(SwiftNativeFileSystemStore store) {
this.store = store;
}
/**
* This is for testing
* @return the inner store class
*/
public SwiftNativeFileSystemStore getStore() {
return store;
}
@Override
public String getScheme() {
return SWIFT;
}
/**
* default class initialization
*
* @param fsuri path to Swift
* @param conf Hadoop configuration
* @throws IOException
*/
@Override
public void initialize(URI fsuri, Configuration conf) throws IOException {
super.initialize(fsuri, conf);
setConf(conf);
if (store == null) {
store = new SwiftNativeFileSystemStore();
}
this.uri = fsuri;
String username = System.getProperty("user.name");
this.workingDir = new Path("/user", username)
.makeQualified(uri, new Path(username));
if (LOG.isDebugEnabled()) {
LOG.debug("Initializing SwiftNativeFileSystem against URI " + uri
+ " and working dir " + workingDir);
}
store.initialize(uri, conf);
LOG.debug("SwiftFileSystem initialized");
}
/**
* @return path to Swift
*/
@Override
public URI getUri() {
return uri;
}
@Override
public String toString() {
return "Swift FileSystem " + store;
}
/**
* Path to user working directory
*
* @return Hadoop path
*/
@Override
public Path getWorkingDirectory() {
return workingDir;
}
/**
* @param dir user working directory
*/
@Override
public void setWorkingDirectory(Path dir) {
workingDir = makeAbsolute(dir);
if (LOG.isDebugEnabled()) {
LOG.debug("SwiftFileSystem.setWorkingDirectory to " + dir);
}
}
/**
* Return a file status object that represents the path.
*
* @param path The path we want information from
* @return a FileStatus object
*/
@Override
public FileStatus getFileStatus(Path path) throws IOException {
Path absolutePath = makeAbsolute(path);
return store.getObjectMetadata(absolutePath);
}
/**
* The blocksize of this filesystem is set by the property
* SwiftProtocolConstants.SWIFT_BLOCKSIZE;the default is the value of
* SwiftProtocolConstants.DEFAULT_SWIFT_BLOCKSIZE;
* @return the blocksize for this FS.
*/
@Override
public long getDefaultBlockSize() {
return store.getBlocksize();
}
/**
* The blocksize for this filesystem.
* @see #getDefaultBlockSize()
* @param f path of file
* @return the blocksize for the path
*/
@Override
public long getDefaultBlockSize(Path f) {
return store.getBlocksize();
}
@Override
public long getBlockSize(Path path) throws IOException {
return store.getBlocksize();
}
@Override
public boolean isFile(Path f) throws IOException {
try {
FileStatus fileStatus = getFileStatus(f);
return !SwiftUtils.isDirectory(fileStatus);
} catch (FileNotFoundException e) {
return false; // f does not exist
}
}
@Override
public boolean isDirectory(Path f) throws IOException {
try {
FileStatus fileStatus = getFileStatus(f);
return SwiftUtils.isDirectory(fileStatus);
} catch (FileNotFoundException e) {
return false; // f does not exist
}
}
/**
* Return an array containing hostnames, offset and size of
* portions of the given file. For a nonexistent
* file or regions, null will be returned.
* <p/>
* This call is most helpful with DFS, where it returns
* hostnames of machines that contain the given file.
* <p/>
* The FileSystem will simply return an elt containing 'localhost'.
*/
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file,
long start,
long len) throws IOException {
//argument checks
if (file == null) {
return null;
}
if (start < 0 || len < 0) {
throw new IllegalArgumentException("Negative start or len parameter" +
" to getFileBlockLocations");
}
if (file.getLen() <= start) {
return new BlockLocation[0];
}
// Check if requested file in Swift is more than 5Gb. In this case
// each block has its own location -which may be determinable
// from the Swift client API, depending on the remote server
final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath(),
false,
true);
List<URI> locations = new ArrayList<URI>();
if (listOfFileBlocks.length > 1) {
for (FileStatus fileStatus : listOfFileBlocks) {
if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
.equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
continue;
}
locations.addAll(store.getObjectLocation(fileStatus.getPath()));
}
} else {
locations = store.getObjectLocation(file.getPath());
}
if (locations.isEmpty()) {
LOG.debug("No locations returned for " + file.getPath());
//no locations were returned for the object
//fall back to the superclass
String[] name = {SwiftProtocolConstants.BLOCK_LOCATION};
String[] host = { "localhost" };
String[] topology={SwiftProtocolConstants.TOPOLOGY_PATH};
return new BlockLocation[] {
new BlockLocation(name, host, topology,0, file.getLen())
};
}
final String[] names = new String[locations.size()];
final String[] hosts = new String[locations.size()];
int i = 0;
for (URI location : locations) {
hosts[i] = location.getHost();
names[i] = location.getAuthority();
i++;
}
return new BlockLocation[]{
new BlockLocation(names, hosts, 0, file.getLen())
};
}
/**
* Create the parent directories.
* As an optimization, the entire hierarchy of parent
* directories is <i>Not</i> polled. Instead
* the tree is walked up from the last to the first,
* creating directories until one that exists is found.
*
* This strategy means if a file is created in an existing directory,
* one quick poll sufficies.
*
* There is a big assumption here: that all parent directories of an existing
* directory also exists.
* @param path path to create.
* @param permission to apply to files
* @return true if the operation was successful
* @throws IOException on a problem
*/
@Override
public boolean mkdirs(Path path, FsPermission permission) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("SwiftFileSystem.mkdirs: " + path);
}
Path directory = makeAbsolute(path);
//build a list of paths to create
List<Path> paths = new ArrayList<Path>();
while (shouldCreate(directory)) {
//this directory needs creation, add to the list
paths.add(0, directory);
//now see if the parent needs to be created
directory = directory.getParent();
}
//go through the list of directories to create
for (Path p : paths) {
if (isNotRoot(p)) {
//perform a mkdir operation without any polling of
//the far end first
forceMkdir(p);
}
}
//if an exception was not thrown, this operation is considered
//a success
return true;
}
private boolean isNotRoot(Path absolutePath) {
return !isRoot(absolutePath);
}
private boolean isRoot(Path absolutePath) {
return absolutePath.getParent() == null;
}
/**
* internal implementation of directory creation.
*
* @param path path to file
* @return boolean file is created; false: no need to create
* @throws IOException if specified path is file instead of directory
*/
private boolean mkdir(Path path) throws IOException {
Path directory = makeAbsolute(path);
boolean shouldCreate = shouldCreate(directory);
if (shouldCreate) {
forceMkdir(directory);
}
return shouldCreate;
}
/**
* Should mkdir create this directory?
* If the directory is root : false
* If the entry exists and is a directory: false
* If the entry exists and is a file: exception
* else: true
* @param directory path to query
* @return true iff the directory should be created
* @throws IOException IO problems
* @throws SwiftNotDirectoryException if the path references a file
*/
private boolean shouldCreate(Path directory) throws IOException {
FileStatus fileStatus;
boolean shouldCreate;
if (isRoot(directory)) {
//its the base dir, bail out immediately
return false;
}
try {
//find out about the path
fileStatus = getFileStatus(directory);
if (!SwiftUtils.isDirectory(fileStatus)) {
//if it's a file, raise an error
throw new SwiftNotDirectoryException(directory,
String.format(": can't mkdir since it exists and is not a directory: %s",
fileStatus));
} else {
//path exists, and it is a directory
if (LOG.isDebugEnabled()) {
LOG.debug("skipping mkdir(" + directory + ") as it exists already");
}
shouldCreate = false;
}
} catch (FileNotFoundException e) {
shouldCreate = true;
}
return shouldCreate;
}
/**
* mkdir of a directory -irrespective of what was there underneath.
* There are no checks for the directory existing, there not
* being a path there, etc. etc. Those are assumed to have
* taken place already
* @param absolutePath path to create
* @throws IOException IO problems
*/
private void forceMkdir(Path absolutePath) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Making dir '" + absolutePath + "' in Swift");
}
//file is not found: it must be created
store.createDirectory(absolutePath);
}
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
*
* @param path given path
* @return the statuses of the files/directories in the given path
* @throws IOException
*/
@Override
public FileStatus[] listStatus(Path path) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("SwiftFileSystem.listStatus for: " + path);
}
return store.listSubPaths(makeAbsolute(path), false, true);
}
/**
* This optional operation is not supported
*/
@Override
public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
throws IOException {
LOG.debug("SwiftFileSystem.append");
throw new SwiftUnsupportedFeatureException("Not supported: append()");
}
/**
* @param permission Currently ignored.
*/
@Override
public FSDataOutputStream create(Path file, FsPermission permission,
boolean overwrite, int bufferSize,
short replication, long blockSize,
Progressable progress)
throws IOException {
LOG.debug("SwiftFileSystem.create");
FileStatus fileStatus = null;
Path absolutePath = makeAbsolute(file);
try {
fileStatus = getFileStatus(absolutePath);
} catch (FileNotFoundException e) {
//the file isn't there.
}
if (fileStatus != null) {
//the path exists -action depends on whether or not it is a directory,
//and what the overwrite policy is.
//What is clear at this point is that if the entry exists, there's
//no need to bother creating any parent entries
if (fileStatus.isDirectory()) {
//here someone is trying to create a file over a directory
/* we can't throw an exception here as there is no easy way to distinguish
a file from the dir
throw new SwiftPathExistsException("Cannot create a file over a directory:"
+ file);
*/
if (LOG.isDebugEnabled()) {
LOG.debug("Overwriting either an empty file or a directory");
}
}
if (overwrite) {
//overwrite set -> delete the object.
store.delete(absolutePath, true);
} else {
throw new SwiftPathExistsException("Path exists: " + file);
}
} else {
// destination does not exist -trigger creation of the parent
Path parent = file.getParent();
if (parent != null) {
if (!mkdirs(parent)) {
throw new SwiftOperationFailedException(
"Mkdirs failed to create " + parent);
}
}
}
SwiftNativeOutputStream out = createSwiftOutputStream(file);
return new FSDataOutputStream(out, statistics);
}
/**
* Create the swift output stream
* @param path path to write to
* @return the new file
* @throws IOException
*/
protected SwiftNativeOutputStream createSwiftOutputStream(Path path) throws
IOException {
long partSizeKB = getStore().getPartsizeKB();
return new SwiftNativeOutputStream(getConf(),
getStore(),
path.toUri().toString(),
partSizeKB);
}
/**
* Opens an FSDataInputStream at the indicated Path.
*
* @param path the file name to open
* @param bufferSize the size of the buffer to be used.
* @return the input stream
* @throws FileNotFoundException if the file is not found
* @throws IOException any IO problem
*/
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
int bufferSizeKB = getStore().getBufferSizeKB();
long readBlockSize = bufferSizeKB * 1024L;
return open(path, bufferSize, readBlockSize);
}
/**
* Low-level operation to also set the block size for this operation
* @param path the file name to open
* @param bufferSize the size of the buffer to be used.
* @param readBlockSize how big should the read blockk/buffer size be?
* @return the input stream
* @throws FileNotFoundException if the file is not found
* @throws IOException any IO problem
*/
public FSDataInputStream open(Path path,
int bufferSize,
long readBlockSize) throws IOException {
if (readBlockSize <= 0) {
throw new SwiftConfigurationException("Bad remote buffer size");
}
Path absolutePath = makeAbsolute(path);
return new FSDataInputStream(
new StrictBufferedFSInputStream(
new SwiftNativeInputStream(store,
statistics,
absolutePath,
readBlockSize),
bufferSize));
}
/**
* Renames Path src to Path dst. On swift this uses copy-and-delete
* and <i>is not atomic</i>.
*
* @param src path
* @param dst path
* @return true if directory renamed, false otherwise
* @throws IOException on problems
*/
@Override
public boolean rename(Path src, Path dst) throws IOException {
try {
store.rename(makeAbsolute(src), makeAbsolute(dst));
//success
return true;
} catch (SwiftOperationFailedException e) {
//downgrade to a failure
return false;
} catch (FileNotFoundException e) {
//downgrade to a failure
return false;
}
}
/**
* Delete a file or directory
*
* @param path the path to delete.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception if the
* directory is not empty
* case of a file the recursive can be set to either true or false.
* @return true if the object was deleted
* @throws IOException IO problems
*/
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
try {
return store.delete(path, recursive);
} catch (FileNotFoundException e) {
//base path was not found.
return false;
}
}
/**
* Delete a file.
* This method is abstract in Hadoop 1.x; in 2.x+ it is non-abstract
* and deprecated
*/
@Override
public boolean delete(Path f) throws IOException {
return delete(f, true);
}
/**
* Makes path absolute
*
* @param path path to file
* @return absolute path
*/
protected Path makeAbsolute(Path path) {
if (path.isAbsolute()) {
return path;
}
return new Path(workingDir, path);
}
/**
* Get the current operation statistics
* @return a snapshot of the statistics
*/
public List<DurationStats> getOperationStatistics() {
return store.getOperationStatistics();
}
/**
* Low level method to do a deep listing of all entries, not stopping
* at the next directory entry. This is to let tests be confident that
* recursive deletes &c really are working.
* @param path path to recurse down
* @param newest ask for the newest data, potentially slower than not.
* @return a potentially empty array of file status
* @throws IOException any problem
*/
@InterfaceAudience.Private
public FileStatus[] listRawFileStatus(Path path, boolean newest) throws IOException {
return store.listSubPaths(makeAbsolute(path), true, newest);
}
/**
* Get the number of partitions written by an output stream
* This is for testing
* @param outputStream output stream
* @return the #of partitions written by that stream
*/
@InterfaceAudience.Private
public static int getPartitionsWritten(FSDataOutputStream outputStream) {
SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
return snos.getPartitionsWritten();
}
private static SwiftNativeOutputStream getSwiftNativeOutputStream(
FSDataOutputStream outputStream) {
OutputStream wrappedStream = outputStream.getWrappedStream();
return (SwiftNativeOutputStream) wrappedStream;
}
/**
* Get the size of partitions written by an output stream
* This is for testing
*
* @param outputStream output stream
* @return partition size in bytes
*/
@InterfaceAudience.Private
public static long getPartitionSize(FSDataOutputStream outputStream) {
SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
return snos.getFilePartSize();
}
/**
* Get the the number of bytes written to an output stream
* This is for testing
*
* @param outputStream output stream
* @return partition size in bytes
*/
@InterfaceAudience.Private
public static long getBytesWritten(FSDataOutputStream outputStream) {
SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
return snos.getBytesWritten();
}
/**
* Get the the number of bytes uploaded by an output stream
* to the swift cluster.
* This is for testing
*
* @param outputStream output stream
* @return partition size in bytes
*/
@InterfaceAudience.Private
public static long getBytesUploaded(FSDataOutputStream outputStream) {
SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
return snos.getBytesUploaded();
}
}

977
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java Normal file
View File

@ -0,0 +1,977 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import org.apache.hadoop.fs.swift.exceptions.SwiftException;
import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
import org.apache.hadoop.fs.swift.http.HttpBodyContent;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.http.SwiftRestClient;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.JSONUtil;
import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import org.codehaus.jackson.map.type.CollectionType;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* File system store implementation.
* Makes REST requests, parses data from responses
*/
public class SwiftNativeFileSystemStore {
private static final Pattern URI_PATTERN = Pattern.compile("\"\\S+?\"");
private static final String PATTERN = "EEE, d MMM yyyy hh:mm:ss zzz";
private static final Log LOG =
LogFactory.getLog(SwiftNativeFileSystemStore.class);
private URI uri;
private SwiftRestClient swiftRestClient;
/**
* Initalize the filesystem store -this creates the REST client binding.
*
* @param fsURI URI of the filesystem, which is used to map to the filesystem-specific
* options in the configuration file
* @param configuration configuration
* @throws IOException on any failure.
*/
public void initialize(URI fsURI, Configuration configuration) throws IOException {
this.uri = fsURI;
this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
}
@Override
public String toString() {
return "SwiftNativeFileSystemStore with "
+ swiftRestClient;
}
/**
* Get the default blocksize of this (bound) filesystem
* @return the blocksize returned for all FileStatus queries,
* which is used by the MapReduce splitter.
*/
public long getBlocksize() {
return 1024L * swiftRestClient.getBlocksizeKB();
}
public long getPartsizeKB() {
return swiftRestClient.getPartSizeKB();
}
public int getBufferSizeKB() {
return swiftRestClient.getBufferSizeKB();
}
public int getThrottleDelay() {
return swiftRestClient.getThrottleDelay();
}
/**
* Upload a file/input stream of a specific length.
*
* @param path destination path in the swift filesystem
* @param inputStream input data. This is closed afterwards, always
* @param length length of the data
* @throws IOException on a problem
*/
public void uploadFile(Path path, InputStream inputStream, long length)
throws IOException {
swiftRestClient.upload(toObjectPath(path), inputStream, length);
}
/**
* Upload part of a larger file.
*
* @param path destination path
* @param partNumber item number in the path
* @param inputStream input data
* @param length length of the data
* @throws IOException on a problem
*/
public void uploadFilePart(Path path, int partNumber,
InputStream inputStream, long length)
throws IOException {
String stringPath = path.toUri().toString();
String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
partNumber);
if (stringPath.endsWith("/")) {
stringPath = stringPath.concat(partitionFilename);
} else {
stringPath = stringPath.concat("/").concat(partitionFilename);
}
swiftRestClient.upload(
new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
inputStream,
length);
}
/**
* Tell the Swift server to expect a multi-part upload by submitting
* a 0-byte file with the X-Object-Manifest header
*
* @param path path of final final
* @throws IOException
*/
public void createManifestForPartUpload(Path path) throws IOException {
String pathString = toObjectPath(path).toString();
if (!pathString.endsWith("/")) {
pathString = pathString.concat("/");
}
if (pathString.startsWith("/")) {
pathString = pathString.substring(1);
}
swiftRestClient.upload(toObjectPath(path),
new ByteArrayInputStream(new byte[0]),
0,
new Header(SwiftProtocolConstants.X_OBJECT_MANIFEST, pathString));
}
/**
* Get the metadata of an object
*
* @param path path
* @return file metadata. -or null if no headers were received back from the server.
* @throws IOException on a problem
* @throws FileNotFoundException if there is nothing at the end
*/
public SwiftFileStatus getObjectMetadata(Path path) throws IOException {
return getObjectMetadata(path, true);
}
/**
* Get the HTTP headers, in case you really need the low-level
* metadata
* @param path path to probe
* @param newest newest or oldest?
* @return the header list
* @throws IOException IO problem
* @throws FileNotFoundException if there is nothing at the end
*/
public Header[] getObjectHeaders(Path path, boolean newest)
throws IOException, FileNotFoundException {
SwiftObjectPath objectPath = toObjectPath(path);
return stat(objectPath, newest);
}
/**
* Get the metadata of an object
*
* @param path path
* @param newest flag to say "set the newest header", otherwise take any entry
* @return file metadata. -or null if no headers were received back from the server.
* @throws IOException on a problem
* @throws FileNotFoundException if there is nothing at the end
*/
public SwiftFileStatus getObjectMetadata(Path path, boolean newest)
throws IOException, FileNotFoundException {
SwiftObjectPath objectPath = toObjectPath(path);
final Header[] headers = stat(objectPath, newest);
//no headers is treated as a missing file
if (headers.length == 0) {
throw new FileNotFoundException("Not Found " + path.toUri());
}
boolean isDir = false;
long length = 0;
long lastModified = 0 ;
for (Header header : headers) {
String headerName = header.getName();
if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) ||
headerName.equals(SwiftProtocolConstants.X_CONTAINER_BYTES_USED)) {
length = 0;
isDir = true;
}
if (SwiftProtocolConstants.HEADER_CONTENT_LENGTH.equals(headerName)) {
length = Long.parseLong(header.getValue());
}
if (SwiftProtocolConstants.HEADER_LAST_MODIFIED.equals(headerName)) {
final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(PATTERN);
try {
lastModified = simpleDateFormat.parse(header.getValue()).getTime();
} catch (ParseException e) {
throw new SwiftException("Failed to parse " + header.toString(), e);
}
}
}
if (lastModified == 0) {
lastModified = System.currentTimeMillis();
}
Path correctSwiftPath = getCorrectSwiftPath(path);
return new SwiftFileStatus(length,
isDir,
1,
getBlocksize(),
lastModified,
correctSwiftPath);
}
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
IOException {
Header[] headers;
if (newest) {
headers = swiftRestClient.headRequest("getObjectMetadata-newest",
objectPath, SwiftRestClient.NEWEST);
} else {
headers = swiftRestClient.headRequest("getObjectMetadata",
objectPath);
}
return headers;
}
/**
* Get the object as an input stream
*
* @param path object path
* @return the input stream -this must be closed to terminate the connection
* @throws IOException IO problems
* @throws FileNotFoundException path doesn't resolve to an object
*/
public HttpBodyContent getObject(Path path) throws IOException {
return swiftRestClient.getData(toObjectPath(path),
SwiftRestClient.NEWEST);
}
/**
* Get the input stream starting from a specific point.
*
* @param path path to object
* @param byteRangeStart starting point
* @param length no. of bytes
* @return an input stream that must be closed
* @throws IOException IO problems
*/
public HttpBodyContent getObject(Path path, long byteRangeStart, long length)
throws IOException {
return swiftRestClient.getData(
toObjectPath(path), byteRangeStart, length);
}
/**
* List a directory.
* This is O(n) for the number of objects in this path.
*
*
*
* @param path working path
* @param listDeep ask for all the data
* @param newest ask for the newest data
* @return Collection of file statuses
* @throws IOException IO problems
* @throws FileNotFoundException if the path does not exist
*/
private List<FileStatus> listDirectory(SwiftObjectPath path,
boolean listDeep,
boolean newest) throws IOException {
final byte[] bytes;
final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
final Path correctSwiftPath = getCorrectSwiftPath(path);
try {
bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep);
} catch (FileNotFoundException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("" +
"File/Directory not found " + path);
}
if (SwiftUtils.isRootDir(path)) {
return Collections.emptyList();
} else {
throw e;
}
} catch (SwiftInvalidResponseException e) {
//bad HTTP error code
if (e.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
//this can come back on a root list if the container is empty
if (SwiftUtils.isRootDir(path)) {
return Collections.emptyList();
} else {
//NO_CONTENT returned on something other than the root directory;
//see if it is there, and convert to empty list or not found
//depending on whether the entry exists.
FileStatus stat = getObjectMetadata(correctSwiftPath, newest);
if (stat.isDirectory()) {
//it's an empty directory. state that
return Collections.emptyList();
} else {
//it's a file -return that as the status
files.add(stat);
return files;
}
}
} else {
//a different status code: rethrow immediately
throw e;
}
}
final CollectionType collectionType = JSONUtil.getJsonMapper().getTypeFactory().
constructCollectionType(List.class, SwiftObjectFileStatus.class);
final List<SwiftObjectFileStatus> fileStatusList =
JSONUtil.toObject(new String(bytes), collectionType);
//this can happen if user lists file /data/files/file
//in this case swift will return empty array
if (fileStatusList.isEmpty()) {
SwiftFileStatus objectMetadata = getObjectMetadata(correctSwiftPath,
newest);
if (objectMetadata.isFile()) {
files.add(objectMetadata);
}
return files;
}
for (SwiftObjectFileStatus status : fileStatusList) {
if (status.getName() != null) {
files.add(new SwiftFileStatus(status.getBytes(),
status.getBytes() == 0,
1,
getBlocksize(),
status.getLast_modified().getTime(),
getCorrectSwiftPath(new Path(status.getName()))));
}
}
return files;
}
/**
* List all elements in this directory
*
*
*
* @param path path to work with
* @param recursive do a recursive get
* @param newest ask for the newest, or can some out of date data work?
* @return the file statuses, or an empty array if there are no children
* @throws IOException on IO problems
* @throws FileNotFoundException if the path is nonexistent
*/
public FileStatus[] listSubPaths(Path path,
boolean recursive,
boolean newest) throws IOException {
final Collection<FileStatus> fileStatuses;
fileStatuses = listDirectory(toDirPath(path), recursive, newest);
return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
}
/**
* Create a directory
*
* @param path path
* @throws IOException
*/
public void createDirectory(Path path) throws IOException {
innerCreateDirectory(toDirPath(path));
}
/**
* The inner directory creation option. This only creates
* the dir at the given path, not any parent dirs.
* @param swiftObjectPath swift object path at which a 0-byte blob should be
* put
* @throws IOException IO problems
*/
private void innerCreateDirectory(SwiftObjectPath swiftObjectPath)
throws IOException {
swiftRestClient.putRequest(swiftObjectPath);
}
private SwiftObjectPath toDirPath(Path path) throws
SwiftConfigurationException {
return SwiftObjectPath.fromPath(uri, path, false);
}
private SwiftObjectPath toObjectPath(Path path) throws
SwiftConfigurationException {
return SwiftObjectPath.fromPath(uri, path);
}
/**
* Try to find the specific server(s) on which the data lives
* @param path path to probe
* @return a possibly empty list of locations
* @throws IOException on problems determining the locations
*/
public List<URI> getObjectLocation(Path path) throws IOException {
final byte[] objectLocation;
objectLocation = swiftRestClient.getObjectLocation(toObjectPath(path));
if (objectLocation == null || objectLocation.length == 0) {
//no object location, return an empty list
return new LinkedList<URI>();
}
return extractUris(new String(objectLocation), path);
}
/**
* deletes object from Swift
*
* @param path path to delete
* @return true if the path was deleted by this specific operation.
* @throws IOException on a failure
*/
public boolean deleteObject(Path path) throws IOException {
SwiftObjectPath swiftObjectPath = toObjectPath(path);
if (!SwiftUtils.isRootDir(swiftObjectPath)) {
return swiftRestClient.delete(swiftObjectPath);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Not deleting root directory entry");
}
return true;
}
}
/**
* deletes a directory from Swift. This is not recursive
*
* @param path path to delete
* @return true if the path was deleted by this specific operation -or
* the path was root and not acted on.
* @throws IOException on a failure
*/
public boolean rmdir(Path path) throws IOException {
return deleteObject(path);
}
/**
* Does the object exist
*
* @param path object path
* @return true if the metadata of an object could be retrieved
* @throws IOException IO problems other than FileNotFound, which
* is downgraded to an object does not exist return code
*/
public boolean objectExists(Path path) throws IOException {
return objectExists(toObjectPath(path));
}
/**
* Does the object exist
*
* @param path swift object path
* @return true if the metadata of an object could be retrieved
* @throws IOException IO problems other than FileNotFound, which
* is downgraded to an object does not exist return code
*/
public boolean objectExists(SwiftObjectPath path) throws IOException {
try {
Header[] headers = swiftRestClient.headRequest("objectExists",
path,
SwiftRestClient.NEWEST);
//no headers is treated as a missing file
return headers.length != 0;
} catch (FileNotFoundException e) {
return false;
}
}
/**
* Rename through copy-and-delete. this is a consequence of the
* Swift filesystem using the path as the hash
* into the Distributed Hash Table, "the ring" of filenames.
* <p/>
* Because of the nature of the operation, it is not atomic.
*
* @param src source file/dir
* @param dst destination
* @throws IOException IO failure
* @throws SwiftOperationFailedException if the rename failed
* @throws FileNotFoundException if the source directory is missing, or
* the parent directory of the destination
*/
public void rename(Path src, Path dst)
throws FileNotFoundException, SwiftOperationFailedException, IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("mv " + src + " " + dst);
}
boolean renamingOnToSelf = src.equals(dst);
SwiftObjectPath srcObject = toObjectPath(src);
SwiftObjectPath destObject = toObjectPath(dst);
if (SwiftUtils.isRootDir(srcObject)) {
throw new SwiftOperationFailedException("cannot rename root dir");
}
final SwiftFileStatus srcMetadata;
srcMetadata = getObjectMetadata(src);
SwiftFileStatus dstMetadata;
try {
dstMetadata = getObjectMetadata(dst);
} catch (FileNotFoundException e) {
//destination does not exist.
LOG.debug("Destination does not exist");
dstMetadata = null;
}
//check to see if the destination parent directory exists
Path srcParent = src.getParent();
Path dstParent = dst.getParent();
//skip the overhead of a HEAD call if the src and dest share the same
//parent dir (in which case the dest dir exists), or the destination
//directory is root, in which case it must also exist
if (dstParent != null && !dstParent.equals(srcParent)) {
try {
getObjectMetadata(dstParent);
} catch (FileNotFoundException e) {
//destination parent doesn't exist; bail out
LOG.debug("destination parent directory " + dstParent + " doesn't exist");
throw e;
}
}
boolean destExists = dstMetadata != null;
boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata);
//calculate the destination
SwiftObjectPath destPath;
//enum the child entries and everything underneath
List<FileStatus> childStats = listDirectory(srcObject, true, true);
boolean srcIsFile = !srcMetadata.isDir();
if (srcIsFile) {
//source is a simple file OR a partitioned file
// outcomes:
// #1 dest exists and is file: fail
// #2 dest exists and is dir: destination path becomes under dest dir
// #3 dest does not exist: use dest as name
if (destExists) {
if (destIsDir) {
//outcome #2 -move to subdir of dest
destPath = toObjectPath(new Path(dst, src.getName()));
} else {
//outcome #1 dest it's a file: fail if differeent
if (!renamingOnToSelf) {
throw new SwiftOperationFailedException(
"cannot rename a file over one that already exists");
} else {
//is mv self self where self is a file. this becomes a no-op
LOG.debug("Renaming file onto self: no-op => success");
return;
}
}
} else {
//outcome #3 -new entry
destPath = toObjectPath(dst);
}
int childCount = childStats.size();
//here there is one of:
// - a single object ==> standard file
// ->
if (childCount == 0) {
copyThenDeleteObject(srcObject, destPath);
} else {
//do the copy
SwiftUtils.debug(LOG, "Source file appears to be partitioned." +
" copying file and deleting children");
copyObject(srcObject, destPath);
for (FileStatus stat : childStats) {
SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
deleteObject(stat.getPath());
}
swiftRestClient.delete(srcObject);
}
} else {
//here the source exists and is a directory
// outcomes (given we know the parent dir exists if we get this far)
// #1 destination is a file: fail
// #2 destination is a directory: create a new dir under that one
// #3 destination doesn't exist: create a new dir with that name
// #3 and #4 are only allowed if the dest path is not == or under src
if (destExists && !destIsDir) {
// #1 destination is a file: fail
throw new SwiftOperationFailedException(
"the source is a directory, but not the destination");
}
Path targetPath;
if (destExists) {
// #2 destination is a directory: create a new dir under that one
targetPath = new Path(dst, src.getName());
} else {
// #3 destination doesn't exist: create a new dir with that name
targetPath = dst;
}
SwiftObjectPath targetObjectPath = toObjectPath(targetPath);
//final check for any recursive operations
if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
//you can't rename a directory onto itself
throw new SwiftOperationFailedException(
"cannot move a directory under itself");
}
LOG.info("mv " + srcObject + " " + targetPath);
logDirectory("Directory to copy ", srcObject, childStats);
// iterative copy of everything under the directory.
// by listing all children this can be done iteratively
// rather than recursively -everything in this list is either a file
// or a 0-byte-len file pretending to be a directory.
String srcURI = src.toUri().toString();
int prefixStripCount = srcURI.length() + 1;
for (FileStatus fileStatus : childStats) {
Path copySourcePath = fileStatus.getPath();
String copySourceURI = copySourcePath.toUri().toString();
String copyDestSubPath = copySourceURI.substring(prefixStripCount);
Path copyDestPath = new Path(targetPath, copyDestSubPath);
if (LOG.isTraceEnabled()) {
//trace to debug some low-level rename path problems; retained
//in case they ever come back.
LOG.trace("srcURI=" + srcURI
+ "; copySourceURI=" + copySourceURI
+ "; copyDestSubPath=" + copyDestSubPath
+ "; copyDestPath=" + copyDestPath);
}
SwiftObjectPath copyDestination = toObjectPath(copyDestPath);
try {
copyThenDeleteObject(toObjectPath(copySourcePath),
copyDestination);
} catch (FileNotFoundException e) {
LOG.info("Skipping rename of " + copySourcePath);
}
//add a throttle delay
throttle();
}
//now rename self. If missing, create the dest directory and warn
if (!SwiftUtils.isRootDir(srcObject)) {
try {
copyThenDeleteObject(srcObject,
targetObjectPath);
} catch (FileNotFoundException e) {
//create the destination directory
LOG.warn("Source directory deleted during rename", e);
innerCreateDirectory(destObject);
}
}
}
}
/**
* Debug action to dump directory statuses to the debug log
*
* @param message explanation
* @param objectPath object path (can be null)
* @param statuses listing output
*/
private void logDirectory(String message, SwiftObjectPath objectPath,
Iterable<FileStatus> statuses) {
if (LOG.isDebugEnabled()) {
LOG.debug(message + ": listing of " + objectPath);
for (FileStatus fileStatus : statuses) {
LOG.debug(fileStatus.getPath());
}
}
}
public void copy(Path srcKey, Path dstKey) throws IOException {
SwiftObjectPath srcObject = toObjectPath(srcKey);
SwiftObjectPath destObject = toObjectPath(dstKey);
swiftRestClient.copyObject(srcObject, destObject);
}
/**
* Copy an object then, if the copy worked, delete it.
* If the copy failed, the source object is not deleted.
*
* @param srcObject source object path
* @param destObject destination object path
* @throws IOException IO problems
*/
private void copyThenDeleteObject(SwiftObjectPath srcObject,
SwiftObjectPath destObject) throws
IOException {
//do the copy
copyObject(srcObject, destObject);
//getting here means the copy worked
swiftRestClient.delete(srcObject);
}
/**
* Copy an object
* @param srcObject source object path
* @param destObject destination object path
* @throws IOException IO problems
*/
private void copyObject(SwiftObjectPath srcObject,
SwiftObjectPath destObject) throws
IOException {
if (srcObject.isEqualToOrParentOf(destObject)) {
throw new SwiftException(
"Can't copy " + srcObject + " onto " + destObject);
}
//do the copy
boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject);
if (!copySucceeded) {
throw new SwiftException("Copy of " + srcObject + " to "
+ destObject + "failed");
}
}
/**
* Take a Hadoop path and return one which uses the URI prefix and authority
* of this FS. It doesn't make a relative path absolute
* @param path path in
* @return path with a URI bound to this FS
* @throws SwiftException URI cannot be created.
*/
public Path getCorrectSwiftPath(Path path) throws
SwiftException {
try {
final URI fullUri = new URI(uri.getScheme(),
uri.getAuthority(),
path.toUri().getPath(),
null,
null);
return new Path(fullUri);
} catch (URISyntaxException e) {
throw new SwiftException("Specified path " + path + " is incorrect", e);
}
}
/**
* Builds a hadoop-Path from a swift path, inserting the URI authority
* of this FS instance
* @param path swift object path
* @return Hadoop path
* @throws SwiftException if the URI couldn't be created.
*/
private Path getCorrectSwiftPath(SwiftObjectPath path) throws
SwiftException {
try {
final URI fullUri = new URI(uri.getScheme(),
uri.getAuthority(),
path.getObject(),
null,
null);
return new Path(fullUri);
} catch (URISyntaxException e) {
throw new SwiftException("Specified path " + path + " is incorrect", e);
}
}
/**
* extracts URIs from json
* @param json json to parse
* @param path path (used in exceptions)
* @return URIs
* @throws SwiftOperationFailedException on any problem parsing the JSON
*/
public static List<URI> extractUris(String json, Path path) throws
SwiftOperationFailedException {
final Matcher matcher = URI_PATTERN.matcher(json);
final List<URI> result = new ArrayList<URI>();
while (matcher.find()) {
final String s = matcher.group();
final String uri = s.substring(1, s.length() - 1);
try {
URI createdUri = URI.create(uri);
result.add(createdUri);
} catch (IllegalArgumentException e) {
//failure to create the URI, which means this is bad JSON. Convert
//to an exception with useful text
throw new SwiftOperationFailedException(
String.format(
"could not convert \"%s\" into a URI." +
" source: %s " +
" first JSON: %s",
uri, path, json.substring(0, 256)));
}
}
return result;
}
/**
* Insert a throttled wait if the throttle delay >0
* @throws InterruptedIOException if interrupted during sleep
*/
public void throttle() throws InterruptedIOException {
int throttleDelay = getThrottleDelay();
if (throttleDelay > 0) {
try {
Thread.sleep(throttleDelay);
} catch (InterruptedException e) {
//convert to an IOE
throw (InterruptedIOException) new InterruptedIOException(e.toString())
.initCause(e);
}
}
}
/**
* Get the current operation statistics
* @return a snapshot of the statistics
*/
public List<DurationStats> getOperationStatistics() {
return swiftRestClient.getOperationStatistics();
}
/**
* Delete the entire tree. This is an internal one with slightly different
* behavior: if an entry is missing, a {@link FileNotFoundException} is
* raised. This lets the caller distinguish a file not found with
* other reasons for failure, so handles race conditions in recursive
* directory deletes better.
* <p/>
* The problem being addressed is: caller A requests a recursive directory
* of directory /dir ; caller B requests a delete of a file /dir/file,
* between caller A enumerating the files contents, and requesting a delete
* of /dir/file. We want to recognise the special case
* "directed file is no longer there" and not convert that into a failure
*
* @param absolutePath the path to delete.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception if the
* directory is not empty
* case of a file the recursive can be set to either true or false.
* @return true if the object was deleted
* @throws IOException IO problems
* @throws FileNotFoundException if a file/dir being deleted is not there -
* this includes entries below the specified path, (if the path is a dir
* and recursive is true)
*/
public boolean delete(Path absolutePath, boolean recursive) throws IOException {
Path swiftPath = getCorrectSwiftPath(absolutePath);
SwiftUtils.debug(LOG, "Deleting path '%s' recursive=%b",
absolutePath,
recursive);
boolean askForNewest = true;
SwiftFileStatus fileStatus = getObjectMetadata(swiftPath, askForNewest);
//ask for the file/dir status, but don't demand the newest, as we
//don't mind if the directory has changed
//list all entries under this directory.
//this will throw FileNotFoundException if the file isn't there
FileStatus[] statuses = listSubPaths(absolutePath, true, askForNewest);
if (statuses == null) {
//the directory went away during the non-atomic stages of the operation.
// Return false as it was not this thread doing the deletion.
SwiftUtils.debug(LOG, "Path '%s' has no status -it has 'gone away'",
absolutePath,
recursive);
return false;
}
int filecount = statuses.length;
SwiftUtils.debug(LOG, "Path '%s' %d status entries'",
absolutePath,
filecount);
if (filecount == 0) {
//it's an empty directory or a path
rmdir(absolutePath);
return true;
}
if (LOG.isDebugEnabled()) {
SwiftUtils.debug(LOG, SwiftUtils.fileStatsToString(statuses, "\n"));
}
if (filecount == 1 && swiftPath.equals(statuses[0].getPath())) {
// 1 entry => simple file and it is the target
//simple file: delete it
SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath);
deleteObject(absolutePath);
return true;
}
//>1 entry implies directory with children. Run through them,
// but first check for the recursive flag and reject it *unless it looks
// like a partitioned file (len > 0 && has children)
if (!fileStatus.isDir()) {
LOG.debug("Multiple child entries but entry has data: assume partitioned");
} else if (!recursive) {
//if there are children, unless this is a recursive operation, fail immediately
throw new SwiftOperationFailedException("Directory " + fileStatus
+ " is not empty: "
+ SwiftUtils.fileStatsToString(
statuses, "; "));
}
//delete the entries. including ourself.
for (FileStatus entryStatus : statuses) {
Path entryPath = entryStatus.getPath();
try {
boolean deleted = deleteObject(entryPath);
if (!deleted) {
SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing",
entryPath);
}
} catch (FileNotFoundException e) {
//the path went away -race conditions.
//do not fail, as the outcome is still OK.
SwiftUtils.debug(LOG, "Path '%s' is no longer present; continuing",
entryPath);
}
throttle();
}
//now delete self
SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath);
deleteObject(absolutePath);
return true;
}
}

379
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java Normal file
View File

@ -0,0 +1,379 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
import org.apache.hadoop.fs.swift.exceptions.SwiftException;
import org.apache.hadoop.fs.swift.http.HttpBodyContent;
import org.apache.hadoop.fs.swift.http.HttpInputStreamWithRelease;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import java.io.EOFException;
import java.io.IOException;
/**
* The input stream from remote Swift blobs.
* The class attempts to be buffer aware, and react to a forward seek operation
* by trying to scan ahead through the current block of data to find it.
* This accelerates some operations that do a lot of seek()/read() actions,
* including work (such as in the MR engine) that do a seek() immediately after
* an open().
*/
class SwiftNativeInputStream extends FSInputStream {
private static final Log LOG = LogFactory.getLog(SwiftNativeInputStream.class);
/**
* range requested off the server: {@value}
*/
private final long bufferSize;
/**
* File nativeStore instance
*/
private final SwiftNativeFileSystemStore nativeStore;
/**
* Hadoop statistics. Used to get info about number of reads, writes, etc.
*/
private final FileSystem.Statistics statistics;
/**
* Data input stream
*/
private HttpInputStreamWithRelease httpStream;
/**
* File path
*/
private final Path path;
/**
* Current position
*/
private long pos = 0;
/**
* Length of the file picked up at start time
*/
private long contentLength = -1;
/**
* Why the stream is closed
*/
private String reasonClosed = "unopened";
/**
* Offset in the range requested last
*/
private long rangeOffset = 0;
public SwiftNativeInputStream(SwiftNativeFileSystemStore storeNative,
FileSystem.Statistics statistics, Path path, long bufferSize)
throws IOException {
this.nativeStore = storeNative;
this.statistics = statistics;
this.path = path;
if (bufferSize <= 0) {
throw new IllegalArgumentException("Invalid buffer size");
}
this.bufferSize = bufferSize;
//initial buffer fill
this.httpStream = storeNative.getObject(path).getInputStream();
//fillBuffer(0);
}
/**
* Move to a new position within the file relative to where the pointer is now.
* Always call from a synchronized clause
* @param offset offset
*/
private synchronized void incPos(int offset) {
pos += offset;
rangeOffset += offset;
SwiftUtils.trace(LOG, "Inc: pos=%d bufferOffset=%d", pos, rangeOffset);
}
/**
* Update the start of the buffer; always call from a sync'd clause
* @param seekPos position sought.
* @param contentLength content length provided by response (may be -1)
*/
private synchronized void updateStartOfBufferPosition(long seekPos,
long contentLength) {
//reset the seek pointer
pos = seekPos;
//and put the buffer offset to 0
rangeOffset = 0;
this.contentLength = contentLength;
SwiftUtils.trace(LOG, "Move: pos=%d; bufferOffset=%d; contentLength=%d",
pos,
rangeOffset,
contentLength);
}
@Override
public synchronized int read() throws IOException {
verifyOpen();
int result = -1;
try {
result = httpStream.read();
} catch (IOException e) {
String msg = "IOException while reading " + path
+ ": " +e + ", attempting to reopen.";
LOG.debug(msg, e);
if (reopenBuffer()) {
result = httpStream.read();
}
}
if (result != -1) {
incPos(1);
}
if (statistics != null && result != -1) {
statistics.incrementBytesRead(1);
}
return result;
}
@Override
public synchronized int read(byte[] b, int off, int len) throws IOException {
SwiftUtils.debug(LOG, "read(buffer, %d, %d)", off, len);
SwiftUtils.validateReadArgs(b, off, len);
int result = -1;
try {
verifyOpen();
result = httpStream.read(b, off, len);
} catch (IOException e) {
//other IO problems are viewed as transient and re-attempted
LOG.info("Received IOException while reading '" + path +
"', attempting to reopen: " + e);
LOG.debug("IOE on read()" + e, e);
if (reopenBuffer()) {
result = httpStream.read(b, off, len);
}
}
if (result > 0) {
incPos(result);
if (statistics != null) {
statistics.incrementBytesRead(result);
}
}
return result;
}
/**
* Re-open the buffer
* @return true iff more data could be added to the buffer
* @throws IOException if not
*/
private boolean reopenBuffer() throws IOException {
innerClose("reopening buffer to trigger refresh");
boolean success = false;
try {
fillBuffer(pos);
success = true;
} catch (EOFException eof) {
//the EOF has been reached
this.reasonClosed = "End of file";
}
return success;
}
/**
* close the stream. After this the stream is not usable -unless and until
* it is re-opened (which can happen on some of the buffer ops)
* This method is thread-safe and idempotent.
*
* @throws IOException on IO problems.
*/
@Override
public synchronized void close() throws IOException {
innerClose("closed");
}
private void innerClose(String reason) throws IOException {
try {
if (httpStream != null) {
reasonClosed = reason;
if (LOG.isDebugEnabled()) {
LOG.debug("Closing HTTP input stream : " + reason);
}
httpStream.close();
}
} finally {
httpStream = null;
}
}
/**
* Assume that the connection is not closed: throws an exception if it is
* @throws SwiftConnectionClosedException
*/
private void verifyOpen() throws SwiftConnectionClosedException {
if (httpStream == null) {
throw new SwiftConnectionClosedException(reasonClosed);
}
}
@Override
public synchronized String toString() {
return "SwiftNativeInputStream" +
" position=" + pos
+ " buffer size = " + bufferSize
+ " "
+ (httpStream != null ? httpStream.toString()
: (" no input stream: " + reasonClosed));
}
/**
* Treats any finalize() call without the input stream being closed
* as a serious problem, logging at error level
* @throws Throwable n/a
*/
@Override
protected void finalize() throws Throwable {
if (httpStream != null) {
LOG.error(
"Input stream is leaking handles by not being closed() properly: "
+ httpStream.toString());
}
}
/**
* Read through the specified number of bytes.
* The implementation iterates a byte a time, which may seem inefficient
* compared to the read(bytes[]) method offered by input streams.
* However, if you look at the code that implements that method, it comes
* down to read() one char at a time -only here the return value is discarded.
*
*<p/>
* This is a no-op if the stream is closed
* @param bytes number of bytes to read.
* @throws IOException IO problems
* @throws SwiftException if a read returned -1.
*/
private int chompBytes(long bytes) throws IOException {
int count = 0;
if (httpStream != null) {
int result;
for (long i = 0; i < bytes; i++) {
result = httpStream.read();
if (result < 0) {
throw new SwiftException("Received error code while chomping input");
}
count ++;
incPos(1);
}
}
return count;
}
/**
* Seek to an offset. If the data is already in the buffer, move to it
* @param targetPos target position
* @throws IOException on any problem
*/
@Override
public synchronized void seek(long targetPos) throws IOException {
if (targetPos < 0) {
throw new IOException("Negative Seek offset not supported");
}
//there's some special handling of near-local data
//as the seek can be omitted if it is in/adjacent
long offset = targetPos - pos;
if (LOG.isDebugEnabled()) {
LOG.debug("Seek to " + targetPos + "; current pos =" + pos
+ "; offset="+offset);
}
if (offset == 0) {
LOG.debug("seek is no-op");
return;
}
if (offset < 0) {
LOG.debug("seek is backwards");
} else if ((rangeOffset + offset < bufferSize)) {
//if the seek is in range of that requested, scan forwards
//instead of closing and re-opening a new HTTP connection
SwiftUtils.debug(LOG,
"seek is within current stream"
+ "; pos= %d ; targetPos=%d; "
+ "offset= %d ; bufferOffset=%d",
pos, targetPos, offset, rangeOffset);
try {
LOG.debug("chomping ");
chompBytes(offset);
} catch (IOException e) {
//this is assumed to be recoverable with a seek -or more likely to fail
LOG.debug("while chomping ",e);
}
if (targetPos - pos == 0) {
LOG.trace("chomping successful");
return;
}
LOG.trace("chomping failed");
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Seek is beyond buffer size of " + bufferSize);
}
}
innerClose("seeking to " + targetPos);
fillBuffer(targetPos);
}
/**
* Fill the buffer from the target position
* If the target position == current position, the
* read still goes ahead; this is a way of handling partial read failures
* @param targetPos target position
* @throws IOException IO problems on the read
*/
private void fillBuffer(long targetPos) throws IOException {
long length = targetPos + bufferSize;
SwiftUtils.debug(LOG, "Fetching %d bytes starting at %d", length, targetPos);
HttpBodyContent blob = nativeStore.getObject(path, targetPos, length);
httpStream = blob.getInputStream();
updateStartOfBufferPosition(targetPos, blob.getContentLength());
}
@Override
public synchronized long getPos() throws IOException {
return pos;
}
/**
* This FS doesn't explicitly support multiple data sources, so
* return false here.
* @param targetPos the desired target position
* @return true if a new source of the data has been set up
* as the source of future reads
* @throws IOException IO problems
*/
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
}

388
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java Normal file
View File

@ -0,0 +1,388 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftException;
import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
/**
* Output stream, buffers data on local disk.
* Writes to Swift on the close() method, unless the
* file is significantly large that it is being written as partitions.
* In this case, the first partition is written on the first write that puts
* data over the partition, as may later writes. The close() then causes
* the final partition to be written, along with a partition manifest.
*/
class SwiftNativeOutputStream extends OutputStream {
public static final int ATTEMPT_LIMIT = 3;
private long filePartSize;
private static final Log LOG =
LogFactory.getLog(SwiftNativeOutputStream.class);
private Configuration conf;
private String key;
private File backupFile;
private OutputStream backupStream;
private SwiftNativeFileSystemStore nativeStore;
private boolean closed;
private int partNumber;
private long blockOffset;
private long bytesWritten;
private long bytesUploaded;
private boolean partUpload = false;
final byte[] oneByte = new byte[1];
/**
* Create an output stream
* @param conf configuration to use
* @param nativeStore native store to write through
* @param key the key to write
* @param partSizeKB the partition size
* @throws IOException
*/
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public SwiftNativeOutputStream(Configuration conf,
SwiftNativeFileSystemStore nativeStore,
String key,
long partSizeKB) throws IOException {
this.conf = conf;
this.key = key;
this.backupFile = newBackupFile();
this.nativeStore = nativeStore;
this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile));
this.partNumber = 1;
this.blockOffset = 0;
this.filePartSize = 1024L * partSizeKB;
}
private File newBackupFile() throws IOException {
File dir = new File(conf.get("hadoop.tmp.dir"));
if (!dir.mkdirs() && !dir.exists()) {
throw new SwiftException("Cannot create Swift buffer directory: " + dir);
}
File result = File.createTempFile("output-", ".tmp", dir);
result.deleteOnExit();
return result;
}
/**
* Flush the local backing stream.
* This does not trigger a flush of data to the remote blobstore.
* @throws IOException
*/
@Override
public void flush() throws IOException {
backupStream.flush();
}
/**
* check that the output stream is open
*
* @throws SwiftException if it is not
*/
private synchronized void verifyOpen() throws SwiftException {
if (closed) {
throw new SwiftException("Output stream is closed");
}
}
/**
* Close the stream. This will trigger the upload of all locally cached
* data to the remote blobstore.
* @throws IOException IO problems uploading the data.
*/
@Override
public synchronized void close() throws IOException {
if (closed) {
return;
}
try {
closed = true;
//formally declare as closed.
backupStream.close();
backupStream = null;
Path keypath = new Path(key);
if (partUpload) {
partUpload(true);
nativeStore.createManifestForPartUpload(keypath);
} else {
uploadOnClose(keypath);
}
} finally {
delete(backupFile);
backupFile = null;
}
assert backupStream == null: "backup stream has been reopened";
}
/**
* Upload a file when closed, either in one go, or, if the file is
* already partitioned, by uploading the remaining partition and a manifest.
* @param keypath key as a path
* @throws IOException IO Problems
*/
private void uploadOnClose(Path keypath) throws IOException {
boolean uploadSuccess = false;
int attempt = 0;
while (!uploadSuccess) {
try {
++attempt;
bytesUploaded += uploadFileAttempt(keypath, attempt);
uploadSuccess = true;
} catch (IOException e) {
LOG.info("Upload failed " + e, e);
if (attempt > ATTEMPT_LIMIT) {
throw e;
}
}
}
}
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
private long uploadFileAttempt(Path keypath, int attempt) throws IOException {
long uploadLen = backupFile.length();
SwiftUtils.debug(LOG, "Closing write of file %s;" +
" localfile=%s of length %d - attempt %d",
key,
backupFile,
uploadLen,
attempt);
nativeStore.uploadFile(keypath,
new FileInputStream(backupFile),
uploadLen);
return uploadLen;
}
@Override
protected void finalize() throws Throwable {
if(!closed) {
LOG.warn("stream not closed");
}
if (backupFile != null) {
LOG.warn("Leaking backing file " + backupFile);
}
}
private void delete(File file) {
if (file != null) {
SwiftUtils.debug(LOG, "deleting %s", file);
if (!file.delete()) {
LOG.warn("Could not delete " + file);
}
}
}
@Override
public void write(int b) throws IOException {
//insert to a one byte array
oneByte[0] = (byte) b;
//then delegate to the array writing routine
write(oneByte, 0, 1);
}
@Override
public synchronized void write(byte[] buffer, int offset, int len) throws
IOException {
//validate args
if (offset < 0 || len < 0 || (offset + len) > buffer.length) {
throw new IndexOutOfBoundsException("Invalid offset/length for write");
}
//validate the output stream
verifyOpen();
SwiftUtils.debug(LOG, " write(offset=%d, len=%d)", offset, len);
// if the size of file is greater than the partition limit
while (blockOffset + len >= filePartSize) {
// - then partition the blob and upload as many partitions
// are needed.
//how many bytes to write for this partition.
int subWriteLen = (int) (filePartSize - blockOffset);
if (subWriteLen < 0 || subWriteLen > len) {
throw new SwiftInternalStateException("Invalid subwrite len: "
+ subWriteLen
+ " -buffer len: " + len);
}
writeToBackupStream(buffer, offset, subWriteLen);
//move the offset along and length down
offset += subWriteLen;
len -= subWriteLen;
//now upload the partition that has just been filled up
// (this also sets blockOffset=0)
partUpload(false);
}
//any remaining data is now written
writeToBackupStream(buffer, offset, len);
}
/**
* Write to the backup stream.
* Guarantees:
* <ol>
* <li>backupStream is open</li>
* <li>blockOffset + len &lt; filePartSize</li>
* </ol>
* @param buffer buffer to write
* @param offset offset in buffer
* @param len length of write.
* @throws IOException backup stream write failing
*/
private void writeToBackupStream(byte[] buffer, int offset, int len) throws
IOException {
assert len >= 0 : "remainder to write is negative";
SwiftUtils.debug(LOG," writeToBackupStream(offset=%d, len=%d)", offset, len);
if (len == 0) {
//no remainder -downgrade to noop
return;
}
//write the new data out to the backup stream
backupStream.write(buffer, offset, len);
//increment the counters
blockOffset += len;
bytesWritten += len;
}
/**
* Upload a single partition. This deletes the local backing-file,
* and re-opens it to create a new one.
* @param closingUpload is this the final upload of an upload
* @throws IOException on IO problems
*/
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
private void partUpload(boolean closingUpload) throws IOException {
if (backupStream != null) {
backupStream.close();
}
if (closingUpload && partUpload && backupFile.length() == 0) {
//skipping the upload if
// - it is close time
// - the final partition is 0 bytes long
// - one part has already been written
SwiftUtils.debug(LOG, "skipping upload of 0 byte final partition");
delete(backupFile);
} else {
partUpload = true;
boolean uploadSuccess = false;
int attempt = 0;
while(!uploadSuccess) {
try {
++attempt;
bytesUploaded += uploadFilePartAttempt(attempt);
uploadSuccess = true;
} catch (IOException e) {
LOG.info("Upload failed " + e, e);
if (attempt > ATTEMPT_LIMIT) {
throw e;
}
}
}
delete(backupFile);
partNumber++;
blockOffset = 0;
if (!closingUpload) {
//if not the final upload, create a new output stream
backupFile = newBackupFile();
backupStream =
new BufferedOutputStream(new FileOutputStream(backupFile));
}
}
}
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
private long uploadFilePartAttempt(int attempt) throws IOException {
long uploadLen = backupFile.length();
SwiftUtils.debug(LOG, "Uploading part %d of file %s;" +
" localfile=%s of length %d - attempt %d",
partNumber,
key,
backupFile,
uploadLen,
attempt);
nativeStore.uploadFilePart(new Path(key),
partNumber,
new FileInputStream(backupFile),
uploadLen);
return uploadLen;
}
/**
* Get the file partition size
* @return the partition size
*/
long getFilePartSize() {
return filePartSize;
}
/**
* Query the number of partitions written
* This is intended for testing
* @return the of partitions already written to the remote FS
*/
synchronized int getPartitionsWritten() {
return partNumber - 1;
}
/**
* Get the number of bytes written to the output stream.
* This should always be less than or equal to bytesUploaded.
* @return the number of bytes written to this stream
*/
long getBytesWritten() {
return bytesWritten;
}
/**
* Get the number of bytes uploaded to remote Swift cluster.
* bytesUploaded -bytesWritten = the number of bytes left to upload
* @return the number of bytes written to the remote endpoint
*/
long getBytesUploaded() {
return bytesUploaded;
}
@Override
public String toString() {
return "SwiftNativeOutputStream{" +
", key='" + key + '\'' +
", backupFile=" + backupFile +
", closed=" + closed +
", filePartSize=" + filePartSize +
", partNumber=" + partNumber +
", blockOffset=" + blockOffset +
", partUpload=" + partUpload +
", nativeStore=" + nativeStore +
", bytesWritten=" + bytesWritten +
", bytesUploaded=" + bytesUploaded +
'}';
}
}

115
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java Normal file
View File

@ -0,0 +1,115 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.snative;
import java.util.Date;
/**
* Java mapping of Swift JSON file status.
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
*/
class SwiftObjectFileStatus {
private long bytes;
private String content_type;
private String hash;
private Date last_modified;
private String name;
private String subdir;
SwiftObjectFileStatus() {
}
SwiftObjectFileStatus(long bytes, String content_type, String hash,
Date last_modified, String name) {
this.bytes = bytes;
this.content_type = content_type;
this.hash = hash;
this.last_modified = last_modified;
this.name = name;
}
public long getBytes() {
return bytes;
}
public void setBytes(long bytes) {
this.bytes = bytes;
}
public String getContent_type() {
return content_type;
}
public void setContent_type(String content_type) {
this.content_type = content_type;
}
public String getHash() {
return hash;
}
public void setHash(String hash) {
this.hash = hash;
}
public Date getLast_modified() {
return last_modified;
}
public void setLast_modified(Date last_modified) {
this.last_modified = last_modified;
}
public String getName() {
return pathToRootPath(name);
}
public void setName(String name) {
this.name = name;
}
public String getSubdir() {
return pathToRootPath(subdir);
}
public void setSubdir(String subdir) {
this.subdir = subdir;
}
/**
* If path doesn't starts with '/'
* method will concat '/'
*
* @param path specified path
* @return root path string
*/
private String pathToRootPath(String path) {
if (path == null) {
return null;
}
if (path.startsWith("/")) {
return path;
}
return "/".concat(path);
}
}

57
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
public class Duration {
private final long started;
private long finished;
public Duration() {
started = time();
finished = started;
}
private long time() {
return System.currentTimeMillis();
}
public void finished() {
finished = time();
}
public String getDurationString() {
return humanTime(value());
}
public static String humanTime(long time) {
long seconds = (time / 1000);
long minutes = (seconds / 60);
return String.format("%d:%02d:%03d", minutes, seconds % 60, time % 1000);
}
@Override
public String toString() {
return getDurationString();
}
public long value() {
return finished -started;
}
}

154
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java Normal file
View File

@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
/**
* Build ongoing statistics from duration data
*/
public class DurationStats {
final String operation;
int n;
long sum;
long min;
long max;
double mean, m2;
/**
* Construct statistics for a given operation.
* @param operation operation
*/
public DurationStats(String operation) {
this.operation = operation;
reset();
}
/**
* construct from anothr stats entry;
* all value are copied.
* @param that the source statistics
*/
public DurationStats(DurationStats that) {
operation = that.operation;
n = that.n;
sum = that.sum;
min = that.min;
max = that.max;
mean = that.mean;
m2 = that.m2;
}
/**
* Add a duration
* @param duration the new duration
*/
public void add(Duration duration) {
add(duration.value());
}
/**
* Add a number
* @param x the number
*/
public void add(long x) {
n++;
sum += x;
double delta = x - mean;
mean += delta / n;
m2 += delta * (x - mean);
if (x < min) {
min = x;
}
if (x > max) {
max = x;
}
}
/**
* Reset the data
*/
public void reset() {
n = 0;
sum = 0;
sum = 0;
min = 10000000;
max = 0;
mean = 0;
m2 = 0;
}
/**
* Get the number of entries sampled
* @return the number of durations added
*/
public int getCount() {
return n;
}
/**
* Get the sum of all durations
* @return all the durations
*/
public long getSum() {
return sum;
}
/**
* Get the arithmetic mean of the aggregate statistics
* @return the arithmetic mean
*/
public double getArithmeticMean() {
return mean;
}
/**
* Variance, sigma^2
* @return variance, or, if no samples are there, 0.
*/
public double getVariance() {
return n > 0 ? (m2 / (n - 1)) : 0;
}
/**
* Get the std deviation, sigma
* @return the stddev, 0 may mean there are no samples.
*/
public double getDeviation() {
double variance = getVariance();
return (variance > 0) ? Math.sqrt(variance) : 0;
}
/**
* Covert to a useful string
* @return a human readable summary
*/
@Override
public String toString() {
return String.format(
"%s count=%d total=%.3fs mean=%.3fs stddev=%.3fs min=%.3fs max=%.3fs",
operation,
n,
sum / 1000.0,
mean / 1000.0,
getDeviation() / 1000000.0,
min / 1000.0,
max / 1000.0);
}
}

77
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Build a duration stats table to which you can add statistics.
* Designed to be multithreaded
*/
public class DurationStatsTable {
private Map<String,DurationStats> statsTable
= new HashMap<String, DurationStats>(6);
/**
* Add an operation
* @param operation operation name
* @param duration duration
*/
public void add(String operation, Duration duration, boolean success) {
DurationStats durationStats;
String key = operation;
if (!success) {
key += "-FAIL";
}
synchronized (this) {
durationStats = statsTable.get(key);
if (durationStats == null) {
durationStats = new DurationStats(key);
statsTable.put(key, durationStats);
}
}
synchronized (durationStats) {
durationStats.add(duration);
}
}
/**
* Get the current duration statistics
* @return a snapshot of the statistics
*/
public synchronized List<DurationStats> getDurationStatistics() {
List<DurationStats> results = new ArrayList<DurationStats>(statsTable.size());
for (DurationStats stat: statsTable.values()) {
results.add(new DurationStats(stat));
}
return results;
}
/**
* reset the values of the statistics. This doesn't delete them, merely zeroes them.
*/
public synchronized void reset() {
for (DurationStats stat : statsTable.values()) {
stat.reset();
}
}
}

130
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java Normal file
View File

@ -0,0 +1,130 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
import org.apache.hadoop.fs.swift.exceptions.SwiftJsonMarshallingException;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.type.CollectionType;
import org.codehaus.jackson.type.TypeReference;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
public class JSONUtil {
private static ObjectMapper jsonMapper = new ObjectMapper();
/**
* Private constructor.
*/
private JSONUtil() {
}
/**
* Converting object to JSON string. If errors appears throw
* MeshinException runtime exception.
*
* @param object The object to convert.
* @return The JSON string representation.
* @throws IOException IO issues
* @throws SwiftJsonMarshallingException failure to generate JSON
*/
public static String toJSON(Object object) throws
IOException {
Writer json = new StringWriter();
try {
jsonMapper.writeValue(json, object);
return json.toString();
} catch (JsonGenerationException e) {
throw new SwiftJsonMarshallingException(e.toString(), e);
} catch (JsonMappingException e) {
throw new SwiftJsonMarshallingException(e.toString(), e);
}
}
/**
* Convert string representation to object. If errors appears throw
* Exception runtime exception.
*
* @param value The JSON string.
* @param klazz The class to convert.
* @return The Object of the given class.
*/
public static <T> T toObject(String value, Class<T> klazz) throws
IOException {
try {
return jsonMapper.readValue(value, klazz);
} catch (JsonGenerationException e) {
throw new SwiftJsonMarshallingException(e.toString()
+ " source: " + value,
e);
} catch (JsonMappingException e) {
throw new SwiftJsonMarshallingException(e.toString()
+ " source: " + value,
e);
}
}
/**
* @param value json string
* @param typeReference class type reference
* @param <T> type
* @return deserialized T object
*/
public static <T> T toObject(String value,
final TypeReference<T> typeReference)
throws IOException {
try {
return jsonMapper.readValue(value, typeReference);
} catch (JsonGenerationException e) {
throw new SwiftJsonMarshallingException("Error generating response", e);
} catch (JsonMappingException e) {
throw new SwiftJsonMarshallingException("Error generating response", e);
}
}
/**
* @param value json string
* @param collectionType class describing how to deserialize collection of objects
* @param <T> type
* @return deserialized T object
*/
public static <T> T toObject(String value,
final CollectionType collectionType)
throws IOException {
try {
return jsonMapper.readValue(value, collectionType);
} catch (JsonGenerationException e) {
throw new SwiftJsonMarshallingException(e.toString()
+ " source: " + value,
e);
} catch (JsonMappingException e) {
throw new SwiftJsonMarshallingException(e.toString()
+ " source: " + value,
e);
}
}
public static ObjectMapper getJsonMapper() {
return jsonMapper;
}
}

183
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java Normal file
View File

@ -0,0 +1,183 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import org.apache.hadoop.fs.swift.http.RestClientBindings;
import java.net.URI;
import java.util.regex.Pattern;
/**
* Swift hierarchy mapping of (container, path)
*/
public final class SwiftObjectPath {
private static final Pattern PATH_PART_PATTERN = Pattern.compile(".*/AUTH_\\w*/");
/**
* Swift container
*/
private final String container;
/**
* swift object
*/
private final String object;
private final String uriPath;
/**
* Build an instance from a (host, object) pair
*
* @param container container name
* @param object object ref underneath the container
*/
public SwiftObjectPath(String container, String object) {
this.container = container;
this.object = object;
uriPath = buildUriPath();
}
public String getContainer() {
return container;
}
public String getObject() {
return object;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof SwiftObjectPath)) return false;
final SwiftObjectPath that = (SwiftObjectPath) o;
return this.toUriPath().equals(that.toUriPath());
}
@Override
public int hashCode() {
int result = container.hashCode();
result = 31 * result + object.hashCode();
return result;
}
private String buildUriPath() {
return SwiftUtils.joinPaths(container, object);
}
public String toUriPath() {
return uriPath;
}
@Override
public String toString() {
return toUriPath();
}
/**
* Test for the object matching a path, ignoring the container
* value.
*
* @param path path string
* @return true iff the object's name matches the path
*/
public boolean objectMatches(String path) {
return object.equals(path);
}
/**
* Query to see if the possibleChild object is a child path of this.
* object.
*
* The test is done by probing for the path of the this object being
* at the start of the second -with a trailing slash, and both
* containers being equal
*
* @param possibleChild possible child dir
* @return true iff the possibleChild is under this object
*/
public boolean isEqualToOrParentOf(SwiftObjectPath possibleChild) {
String origPath = toUriPath();
String path = origPath;
if (!path.endsWith("/")) {
path = path + "/";
}
String childPath = possibleChild.toUriPath();
return childPath.equals(origPath) || childPath.startsWith(path);
}
/**
* Create a path tuple of (container, path), where the container is
* chosen from the host of the URI.
*
* @param uri uri to start from
* @param path path underneath
* @return a new instance.
* @throws SwiftConfigurationException if the URI host doesn't parse into
* container.service
*/
public static SwiftObjectPath fromPath(URI uri,
Path path)
throws SwiftConfigurationException {
return fromPath(uri, path, false);
}
/**
* Create a path tuple of (container, path), where the container is
* chosen from the host of the URI.
* A trailing slash can be added to the path. This is the point where
* these /-es need to be appended, because when you construct a {@link Path}
* instance, {@link Path#normalizePath(String, String)} is called
* -which strips off any trailing slash.
*
* @param uri uri to start from
* @param path path underneath
* @param addTrailingSlash should a trailing slash be added if there isn't one.
* @return a new instance.
* @throws SwiftConfigurationException if the URI host doesn't parse into
* container.service
*/
public static SwiftObjectPath fromPath(URI uri,
Path path,
boolean addTrailingSlash)
throws SwiftConfigurationException {
String url =
path.toUri().getPath().replaceAll(PATH_PART_PATTERN.pattern(), "");
//add a trailing slash if needed
if (addTrailingSlash && !url.endsWith("/")) {
url += "/";
}
String container = uri.getHost();
if (container == null) {
//no container, not good: replace with ""
container = "";
} else if (container.contains(".")) {
//its a container.service URI. Strip the container
container = RestClientBindings.extractContainerName(container);
}
return new SwiftObjectPath(container, url);
}
}

544
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java Normal file
View File

@ -0,0 +1,544 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import org.junit.internal.AssumptionViolatedException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Properties;
/**
* Utilities used across test cases
*/
public class SwiftTestUtils extends org.junit.Assert {
private static final Log LOG =
LogFactory.getLog(SwiftTestUtils.class);
public static final String TEST_FS_SWIFT = "test.fs.swift.name";
public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
/**
* Get the test URI
* @param conf configuration
* @throws SwiftConfigurationException missing parameter or bad URI
*/
public static URI getServiceURI(Configuration conf) throws
SwiftConfigurationException {
String instance = conf.get(TEST_FS_SWIFT);
if (instance == null) {
throw new SwiftConfigurationException(
"Missing configuration entry " + TEST_FS_SWIFT);
}
try {
return new URI(instance);
} catch (URISyntaxException e) {
throw new SwiftConfigurationException("Bad URI: " + instance);
}
}
public static boolean hasServiceURI(Configuration conf) {
String instance = conf.get(TEST_FS_SWIFT);
return instance != null;
}
/**
* Assert that a property in the property set matches the expected value
* @param props property set
* @param key property name
* @param expected expected value. If null, the property must not be in the set
*/
public static void assertPropertyEquals(Properties props,
String key,
String expected) {
String val = props.getProperty(key);
if (expected == null) {
assertNull("Non null property " + key + " = " + val, val);
} else {
assertEquals("property " + key + " = " + val,
expected,
val);
}
}
/**
*
* Write a file and read it in, validating the result. Optional flags control
* whether file overwrite operations should be enabled, and whether the
* file should be deleted afterwards.
*
* If there is a mismatch between what was written and what was expected,
* a small range of bytes either side of the first error are logged to aid
* diagnosing what problem occurred -whether it was a previous file
* or a corrupting of the current file. This assumes that two
* sequential runs to the same path use datasets with different character
* moduli.
*
* @param fs filesystem
* @param path path to write to
* @param len length of data
* @param overwrite should the create option allow overwrites?
* @param delete should the file be deleted afterwards? -with a verification
* that it worked. Deletion is not attempted if an assertion has failed
* earlier -it is not in a <code>finally{}</code> block.
* @throws IOException IO problems
*/
public static void writeAndRead(FileSystem fs,
Path path,
byte[] src,
int len,
int blocksize,
boolean overwrite,
boolean delete) throws IOException {
fs.mkdirs(path.getParent());
writeDataset(fs, path, src, len, blocksize, overwrite);
byte[] dest = readDataset(fs, path, len);
compareByteArrays(src, dest, len);
if (delete) {
boolean deleted = fs.delete(path, false);
assertTrue("Deleted", deleted);
assertPathDoesNotExist(fs, "Cleanup failed", path);
}
}
/**
* Write a file.
* Optional flags control
* whether file overwrite operations should be enabled
* @param fs filesystem
* @param path path to write to
* @param len length of data
* @param overwrite should the create option allow overwrites?
* @throws IOException IO problems
*/
public static void writeDataset(FileSystem fs,
Path path,
byte[] src,
int len,
int blocksize,
boolean overwrite) throws IOException {
assertTrue(
"Not enough data in source array to write " + len + " bytes",
src.length >= len);
FSDataOutputStream out = fs.create(path,
overwrite,
fs.getConf()
.getInt(IO_FILE_BUFFER_SIZE,
4096),
(short) 1,
blocksize);
out.write(src, 0, len);
out.close();
assertFileHasLength(fs, path, len);
}
/**
* Read the file and convert to a byte dataaset
* @param fs filesystem
* @param path path to read from
* @param len length of data to read
* @return the bytes
* @throws IOException IO problems
*/
public static byte[] readDataset(FileSystem fs, Path path, int len)
throws IOException {
FSDataInputStream in = fs.open(path);
byte[] dest = new byte[len];
try {
in.readFully(0, dest);
} finally {
in.close();
}
return dest;
}
/**
* Assert that tthe array src[0..len] and dest[] are equal
* @param src source data
* @param dest actual
* @param len length of bytes to compare
*/
public static void compareByteArrays(byte[] src,
byte[] dest,
int len) {
assertEquals("Number of bytes read != number written",
len, dest.length);
int errors = 0;
int first_error_byte = -1;
for (int i = 0; i < len; i++) {
if (src[i] != dest[i]) {
if (errors == 0) {
first_error_byte = i;
}
errors++;
}
}
if (errors > 0) {
String message = String.format(" %d errors in file of length %d",
errors, len);
LOG.warn(message);
// the range either side of the first error to print
// this is a purely arbitrary number, to aid user debugging
final int overlap = 10;
for (int i = Math.max(0, first_error_byte - overlap);
i < Math.min(first_error_byte + overlap, len);
i++) {
byte actual = dest[i];
byte expected = src[i];
String letter = toChar(actual);
String line = String.format("[%04d] %2x %s\n", i, actual, letter);
if (expected != actual) {
line = String.format("[%04d] %2x %s -expected %2x %s\n",
i,
actual,
letter,
expected,
toChar(expected));
}
LOG.warn(line);
}
fail(message);
}
}
/**
* Convert a byte to a character for printing. If the
* byte value is < 32 -and hence unprintable- the byte is
* returned as a two digit hex value
* @param b byte
* @return the printable character string
*/
public static String toChar(byte b) {
if (b >= 0x20) {
return Character.toString((char) b);
} else {
return String.format("%02x", b);
}
}
public static String toChar(byte[] buffer) {
StringBuilder builder = new StringBuilder(buffer.length);
for (byte b : buffer) {
builder.append(toChar(b));
}
return builder.toString();
}
public static byte[] toAsciiByteArray(String s) {
char[] chars = s.toCharArray();
int len = chars.length;
byte[] buffer = new byte[len];
for (int i = 0; i < len; i++) {
buffer[i] = (byte) (chars[i] & 0xff);
}
return buffer;
}
public static void cleanupInTeardown(FileSystem fileSystem,
String cleanupPath) {
cleanup("TEARDOWN", fileSystem, cleanupPath);
}
public static void cleanup(String action,
FileSystem fileSystem,
String cleanupPath) {
noteAction(action);
try {
if (fileSystem != null) {
fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem),
true);
}
} catch (Exception e) {
LOG.error("Error deleting in "+ action + " - " + cleanupPath + ": " + e, e);
}
}
public static void noteAction(String action) {
if (LOG.isDebugEnabled()) {
LOG.debug("============== "+ action +" =============");
}
}
/**
* downgrade a failure to a message and a warning, then an
* exception for the Junit test runner to mark as failed
* @param message text message
* @param failure what failed
* @throws AssumptionViolatedException always
*/
public static void downgrade(String message, Throwable failure) {
LOG.warn("Downgrading test " + message, failure);
AssumptionViolatedException ave =
new AssumptionViolatedException(failure, null);
throw ave;
}
/**
* report an overridden test as unsupported
* @param message message to use in the text
* @throws AssumptionViolatedException always
*/
public static void unsupported(String message) {
throw new AssumptionViolatedException(message);
}
/**
* report a test has been skipped for some reason
* @param message message to use in the text
* @throws AssumptionViolatedException always
*/
public static void skip(String message) {
throw new AssumptionViolatedException(message);
}
/**
* Make an assertion about the length of a file
* @param fs filesystem
* @param path path of the file
* @param expected expected length
* @throws IOException on File IO problems
*/
public static void assertFileHasLength(FileSystem fs, Path path,
int expected) throws IOException {
FileStatus status = fs.getFileStatus(path);
assertEquals(
"Wrong file length of file " + path + " status: " + status,
expected,
status.getLen());
}
/**
* Assert that a path refers to a directory
* @param fs filesystem
* @param path path of the directory
* @throws IOException on File IO problems
*/
public static void assertIsDirectory(FileSystem fs,
Path path) throws IOException {
FileStatus fileStatus = fs.getFileStatus(path);
assertIsDirectory(fileStatus);
}
/**
* Assert that a path refers to a directory
* @param fileStatus stats to check
*/
public static void assertIsDirectory(FileStatus fileStatus) {
assertTrue("Should be a dir -but isn't: " + fileStatus,
fileStatus.isDirectory());
}
/**
* Write the text to a file, returning the converted byte array
* for use in validating the round trip
* @param fs filesystem
* @param path path of file
* @param text text to write
* @param overwrite should the operation overwrite any existing file?
* @return the read bytes
* @throws IOException on IO problems
*/
public static byte[] writeTextFile(FileSystem fs,
Path path,
String text,
boolean overwrite) throws IOException {
FSDataOutputStream stream = fs.create(path, overwrite);
byte[] bytes = new byte[0];
if (text != null) {
bytes = toAsciiByteArray(text);
stream.write(bytes);
}
stream.close();
return bytes;
}
/**
* Touch a file: fails if it is already there
* @param fs filesystem
* @param path path
* @throws IOException IO problems
*/
public static void touch(FileSystem fs,
Path path) throws IOException {
fs.delete(path, true);
writeTextFile(fs, path, null, false);
}
public static void assertDeleted(FileSystem fs,
Path file,
boolean recursive) throws IOException {
assertPathExists(fs, "about to be deleted file", file);
boolean deleted = fs.delete(file, recursive);
String dir = ls(fs, file.getParent());
assertTrue("Delete failed on " + file + ": " + dir, deleted);
assertPathDoesNotExist(fs, "Deleted file", file);
}
/**
* Read in "length" bytes, convert to an ascii string
* @param fs filesystem
* @param path path to read
* @param length #of bytes to read.
* @return the bytes read and converted to a string
* @throws IOException
*/
public static String readBytesToString(FileSystem fs,
Path path,
int length) throws IOException {
FSDataInputStream in = fs.open(path);
try {
byte[] buf = new byte[length];
in.readFully(0, buf);
return toChar(buf);
} finally {
in.close();
}
}
public static String getDefaultWorkingDirectory() {
return "/user/" + System.getProperty("user.name");
}
public static String ls(FileSystem fileSystem, Path path) throws IOException {
return SwiftUtils.ls(fileSystem, path);
}
public static String dumpStats(String pathname, FileStatus[] stats) {
return pathname + SwiftUtils.fileStatsToString(stats,"\n");
}
/**
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
* @param fileSystem filesystem to resolve path against
* @param filename name of the file
* @throws IOException IO problems during file operations
*/
public static void assertIsFile(FileSystem fileSystem, Path filename) throws
IOException {
assertPathExists(fileSystem, "Expected file", filename);
FileStatus status = fileSystem.getFileStatus(filename);
String fileInfo = filename + " " + status;
assertFalse("File claims to be a directory " + fileInfo,
status.isDirectory());
/* disabled for Hadoop v1 compatibility
assertFalse("File claims to be a symlink " + fileInfo,
status.isSymlink());
*/
}
/**
* Create a dataset for use in the tests; all data is in the range
* base to (base+modulo-1) inclusive
* @param len length of data
* @param base base of the data
* @param modulo the modulo
* @return the newly generated dataset
*/
public static byte[] dataset(int len, int base, int modulo) {
byte[] dataset = new byte[len];
for (int i = 0; i < len; i++) {
dataset[i] = (byte) (base + (i % modulo));
}
return dataset;
}
/**
* Assert that a path exists -but make no assertions as to the
* type of that entry
*
* @param fileSystem filesystem to examine
* @param message message to include in the assertion failure message
* @param path path in the filesystem
* @throws IOException IO problems
*/
public static void assertPathExists(FileSystem fileSystem, String message,
Path path) throws IOException {
if (!fileSystem.exists(path)) {
//failure, report it
fail(message + ": not found " + path + " in " + path.getParent());
ls(fileSystem, path.getParent());
}
}
/**
* Assert that a path does not exist
*
* @param fileSystem filesystem to examine
* @param message message to include in the assertion failure message
* @param path path in the filesystem
* @throws IOException IO problems
*/
public static void assertPathDoesNotExist(FileSystem fileSystem,
String message,
Path path) throws IOException {
try {
FileStatus status = fileSystem.getFileStatus(path);
fail(message + ": unexpectedly found " + path + " as " + status);
} catch (FileNotFoundException expected) {
//this is expected
}
}
/**
* Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
* @param fs filesystem
* @param dir directory to scan
* @param subdir full path to look for
* @throws IOException IO probles
*/
public static void assertListStatusFinds(FileSystem fs,
Path dir,
Path subdir) throws IOException {
FileStatus[] stats = fs.listStatus(dir);
boolean found = false;
StringBuilder builder = new StringBuilder();
for (FileStatus stat : stats) {
builder.append(stat.toString()).append('\n');
if (stat.getPath().equals(subdir)) {
found = true;
}
}
assertTrue("Path " + subdir
+ " not found in directory " + dir + ":" + builder,
found);
}
}

216
hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java Normal file
View File

@ -0,0 +1,216 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.util;
import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.FileNotFoundException;
import java.io.IOException;
/**
* Various utility classes for SwiftFS support
*/
public final class SwiftUtils {
public static final String READ = "read(buffer, offset, length)";
/**
* Join two (non null) paths, inserting a forward slash between them
* if needed
*
* @param path1 first path
* @param path2 second path
* @return the combined path
*/
public static String joinPaths(String path1, String path2) {
StringBuilder result =
new StringBuilder(path1.length() + path2.length() + 1);
result.append(path1);
boolean insertSlash = true;
if (path1.endsWith("/")) {
insertSlash = false;
} else if (path2.startsWith("/")) {
insertSlash = false;
}
if (insertSlash) {
result.append("/");
}
result.append(path2);
return result.toString();
}
/**
* This test contains the is-directory logic for Swift, so if
* changed there is only one place for it.
*
* @param fileStatus status to examine
* @return true if we consider this status to be representative of a
* directory.
*/
public static boolean isDirectory(FileStatus fileStatus) {
return fileStatus.isDirectory() || isFilePretendingToBeDirectory(fileStatus);
}
/**
* Test for the entry being a file that is treated as if it is a
* directory
*
* @param fileStatus status
* @return true if it meets the rules for being a directory
*/
public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
return fileStatus.getLen() == 0;
}
/**
* Predicate: Is a swift object referring to the root direcory?
* @param swiftObject object to probe
* @return true iff the object refers to the root
*/
public static boolean isRootDir(SwiftObjectPath swiftObject) {
return swiftObject.objectMatches("") || swiftObject.objectMatches("/");
}
/**
* Sprintf() to the log iff the log is at debug level. If the log
* is not at debug level, the printf operation is skipped, so
* no time is spent generating the string.
* @param log log to use
* @param text text message
* @param args args arguments to the print statement
*/
public static void debug(Log log, String text, Object... args) {
if (log.isDebugEnabled()) {
log.debug(String.format(text, args));
}
}
/**
* Log an exception (in text and trace) iff the log is at debug
* @param log Log to use
* @param text text message
* @param ex exception
*/
public static void debugEx(Log log, String text, Exception ex) {
if (log.isDebugEnabled()) {
log.debug(text + ex, ex);
}
}
/**
* Sprintf() to the log iff the log is at trace level. If the log
* is not at trace level, the printf operation is skipped, so
* no time is spent generating the string.
* @param log log to use
* @param text text message
* @param args args arguments to the print statement
*/
public static void trace(Log log, String text, Object... args) {
if (log.isTraceEnabled()) {
log.trace(String.format(text, args));
}
}
/**
* Given a partition number, calculate the partition value.
* This is used in the SwiftNativeOutputStream, and is placed
* here for tests to be able to calculate the filename of
* a partition.
* @param partNumber part number
* @return a string to use as the filename
*/
public static String partitionFilenameFromNumber(int partNumber) {
return String.format("%06d", partNumber);
}
/**
* List a a path to string
* @param fileSystem filesystem
* @param path directory
* @return a listing of the filestatuses of elements in the directory, one
* to a line, precedeed by the full path of the directory
* @throws IOException connectivity problems
*/
public static String ls(FileSystem fileSystem, Path path) throws
IOException {
if (path == null) {
//surfaces when someone calls getParent() on something at the top of the path
return "/";
}
FileStatus[] stats;
String pathtext = "ls " + path;
try {
stats = fileSystem.listStatus(path);
} catch (FileNotFoundException e) {
return pathtext + " -file not found";
} catch (IOException e) {
return pathtext + " -failed: " + e;
}
return pathtext + fileStatsToString(stats, "\n");
}
/**
* Take an array of filestats and convert to a string (prefixed w/ a [01] counter
* @param stats array of stats
* @param separator separator after every entry
* @return a stringified set
*/
public static String fileStatsToString(FileStatus[] stats, String separator) {
StringBuilder buf = new StringBuilder(stats.length * 128);
for (int i = 0; i < stats.length; i++) {
buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
}
return buf.toString();
}
/**
* Verify that the basic args to a read operation are valid;
* throws an exception if not -with meaningful text includeing
* @param buffer destination buffer
* @param off offset
* @param len number of bytes to read
* @throws NullPointerException null buffer
* @throws IndexOutOfBoundsException on any invalid range.
*/
public static void validateReadArgs(byte[] buffer, int off, int len) {
if (buffer == null) {
throw new NullPointerException("Null byte array in"+ READ);
}
if (off < 0 ) {
throw new IndexOutOfBoundsException("Negative buffer offset "
+ off
+ " in " + READ);
}
if (len < 0 ) {
throw new IndexOutOfBoundsException("Negative read length "
+ len
+ " in " + READ);
}
if (off > buffer.length) {
throw new IndexOutOfBoundsException("Buffer offset of "
+ off
+ "beyond buffer size of "
+ buffer.length
+ " in " + READ);
}
}
}

686
hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm Normal file
View File

@ -0,0 +1,686 @@
~~ Licensed under the Apache License, Version 2.0 (the "License");
~~ you may not use this file except in compliance with the License.
~~ You may obtain a copy of the License at
~~
~~ http://www.apache.org/licenses/LICENSE-2.0
~~
~~ Unless required by applicable law or agreed to in writing, software
~~ distributed under the License is distributed on an "AS IS" BASIS,
~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~~ See the License for the specific language governing permissions and
~~ limitations under the License. See accompanying LICENSE file.
---
Hadoop OpenStack Support: Swift Object Store
---
---
${maven.build.timestamp}
%{toc|section=1|fromDepth=0}
Hadoop OpenStack Support: Swift Object Store
* {Introduction}
{{{http://www.openstack.org/}OpenStack}} is an open source cloud infrastructure
which can be accessed
from multiple public IaaS providers, and deployed privately. It offers
infrastructure services such as VM hosting (Nova), authentication (Keystone)
and storage of binary objects (Swift).
This module enables Apache Hadoop applications -including MapReduce jobs,
read and write data to and from instances of the
{{{http://www.openstack.org/software/openstack-storage/}OpenStack Swift object store}}.
* Features
* Read and write of data stored in a Swift object store
* Support of a pseudo-hierachical file system (directories, subdirectories and
files)
* Standard filesystem operations: <<<create>>>, <<<delete>>>, <<<mkdir>>>,
<<<ls>>>, <<<mv>>>, <<<stat>>>.
* Can act as a source of data in a MapReduce job, or a sink.
* Support for multiple OpenStack services, and multiple containers from a
single service.
* Supports in-cluster and remote access to Swift data.
* Supports OpenStack Keystone authentication with password or token.
* Released under the Apache Software License
* Tested against the Hadoop 3.x and 1.x branches, against multiple public
OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
* Tested against private OpenStack clusters, including scalability tests of
large file uploads.
* Using the Hadoop Swift Filesystem Client
** Concepts: services and containers
OpenStack swift is an <Object Store>; also known as a <blobstore>. It stores
arbitrary binary objects by name in a <container>.
The Hadoop Swift filesystem library adds another concept, the <service>, which
defines which Swift blobstore hosts a container -and how to connect to it.
** Containers and Objects
* Containers are created by users with accounts on the Swift filestore, and hold
<objects>.
* Objects can be zero bytes long, or they can contain data.
* Objects in the container can be up to 5GB; there is a special support for
larger files than this, which merges multiple objects in to one.
* Each object is referenced by it's <name>; there is no notion of directories.
* You can use any characters in an object name that can be 'URL-encoded'; the
maximum length of a name is 1034 characters -after URL encoding.
* Names can have <<</>>> characters in them, which are used to create the illusion of
a directory structure. For example <<<dir/dir2/name>>>. Even though this looks
like a directory, <it is still just a name>. There is no requirement to have
any entries in the container called <<<dir>>> or <<<dir/dir2>>>
* That said. if the container has zero-byte objects that look like directory
names above other objects, they can pretend to be directories. Continuing the
example, a 0-byte object called <<<dir>>> would tell clients that it is a
directory while <<<dir/dir2>>> or <<<dir/dir2/name>>> were present. This creates an
illusion of containers holding a filesystem.
Client applications talk to Swift over HTTP or HTTPS, reading, writing and
deleting objects using standard HTTP operations (GET, PUT and DELETE,
respectively). There is also a COPY operation, that creates a new object in the
container, with a new name, containing the old data. There is no rename
operation itself, objects need to be copied -then the original entry deleted.
** Eventual Consistency
The Swift Filesystem is *eventually consistent*: an operation on an object may
not be immediately visible to that client, or other clients. This is a
consequence of the goal of the filesystem: to span a set of machines, across
multiple datacenters, in such a way that the data can still be available when
many of them fail. (In contrast, the Hadoop HDFS filesystem is *immediately
consistent*, but it does not span datacenters.)
Eventual consistency can cause surprises for client applications that expect
immediate consistency: after an object is deleted or overwritten, the object
may still be visible -or the old data still retrievable. The Swift Filesystem
client for Apache Hadoop attempts to handle this, in conjunction with the
MapReduce engine, but there may be still be occasions when eventual consistency
causes surprises.
** Non-atomic "directory" operations.
Hadoop expects some
operations to be atomic, especially <<<rename()>>>, which is something
the MapReduce layer relies on to commit the output of a job, renaming data
from a temp directory to the final path. Because a rename
is implemented as a copy of every blob under the directory's path, followed
by a delete of the originals, the intermediate state of the operation
will be visible to other clients. If two Reducer tasks to rename their temp
directory to the final path, both operations may succeed, with the result that
output directory contains mixed data. This can happen if MapReduce jobs
are being run with <speculation> enabled and Swift used as the direct output
of the MR job (it can also happen against Amazon S3).
Other consequences of the non-atomic operations are:
1. If a program is looking for the presence of the directory before acting
on the data -it may start prematurely. This can be avoided by using
other mechanisms to co-ordinate the programs, such as the presence of a file
that is written <after> any bulk directory operations.
2. A <<<rename()>>> or <<<delete()>>> operation may include files added under
the source directory tree during the operation, may unintentionally delete
it, or delete the 0-byte swift entries that mimic directories and act
as parents for the files. Try to avoid doing this.
The best ways to avoid all these problems is not using Swift as
the filesystem between MapReduce jobs or other Hadoop workflows. It
can act as a source of data, and a final destination, but it doesn't meet
all of Hadoop's expectations of what a filesystem is -it's a <blobstore>.
* Working with Swift Object Stores in Hadoop
Once installed, the Swift FileSystem client can be used by any Hadoop application
to read from or write to data stored in a Swift container.
Data stored in Swift can be used as the direct input to a MapReduce job
-simply use the <<<swift:>>> URL (see below) to declare the source of the data.
This Swift Filesystem client is designed to work with multiple
Swift object stores, both public and private. This allows the client to work
with different clusters, reading and writing data to and from either of them.
It can also work with the same object stores using multiple login details.
These features are achieved by one basic concept: using a service name in
the URI referring to a swift filesystem, and looking up all the connection and
login details for that specific service. Different service names can be defined
in the Hadoop XML configuration file, so defining different clusters, or
providing different login details for the same object store(s).
** Swift Filesystem URIs
Hadoop uses URIs to refer to files within a filesystem. Some common examples
are:
+--
local://etc/hosts
hdfs://cluster1/users/example/data/set1
hdfs://cluster2.example.org:8020/users/example/data/set1
+--
The Swift Filesystem Client adds a new URL type <<<swift>>>. In a Swift Filesystem
URL, the hostname part of a URL identifies the container and the service to
work with; the path the name of the object. Here are some examples
+--
swift://container.rackspace/my-object.csv
swift://data.hpcloud/data/set1
swift://dmitry.privatecloud/out/results
+--
In the last two examples, the paths look like directories: it is not, they are
simply the objects named <<<data/set1>>> and <<<out/results>>> respectively.
** Installing
The <<<hadoop-openstack>>> JAR must be on the classpath of the Hadoop program trying to
talk to the Swift service. If installed in the classpath of the Hadoop
MapReduce service, then all programs started by the MR engine will pick up the
JAR automatically. This is the easiest way to give all Hadoop jobs access to
Swift.
Alternatively, the JAR can be included as one of the JAR files that an
application uses. This lets the Hadoop jobs work with a Swift object store even
if the Hadoop cluster is not pre-configured for this.
The library also depends upon the Apache HttpComponents library, which
must also be on the classpath.
** Configuring
To talk to a swift service, the user must must provide:
[[1]] The URL defining the container and the service.
[[1]] In the cluster/job configuration, the login details of that service.
Multiple service definitions can co-exist in the same configuration file: just
use different names for them.
*** Example: Rackspace US, in-cluster access using API key
This service definition is for use in a Hadoop cluster deployed within Rackspace's
US infrastructure.
+--
<property>
<name>fs.swift.service.rackspace.auth.url</name>
<value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
<description>Rackspace US (multiregion)</description>
</property>
<property>
<name>fs.swift.service.rackspace.username</name>
<value>user4</value>
</property>
<property>
<name>fs.swift.service.rackspace.region</name>
<value>DFW</value>
</property>
<property>
<name>fs.swift.service.rackspace.apikey</name>
<value>fe806aa86dfffe2f6ed8</value>
</property>
+--
Here the API key visible in the account settings API keys page is used to log
in. No property for public/private access -the default is to use the private
endpoint for Swift operations.
This configuration also selects one of the regions, DFW, for its data.
A reference to this service would use the <<<rackspace>>> service name:
---
swift://hadoop-container.rackspace/
---
*** Example: Rackspace UK: remote access with password authentication
This connects to Rackspace's UK ("LON") datacenter.
+--
<property>
<name>fs.swift.service.rackspaceuk.auth.url</name>
<value>https://lon.identity.api.rackspacecloud.com/v2.0/tokens</value>
<description>Rackspace UK</description>
</property>
<property>
<name>fs.swift.service.rackspaceuk.username</name>
<value>user4</value>
</property>
<property>
<name>fs.swift.service.rackspaceuk.password</name>
<value>insert-password-here/value>
</property>
<property>
<name>fs.swift.service.rackspace.public</name>
<value>true</value>
</property>
+--
This is a public access point connection, using a password over an API key.
A reference to this service would use the <<<rackspaceuk>>> service name:
+--
swift://hadoop-container.rackspaceuk/
+--
Because the public endpoint is used, if this service definition is used within
the London datacenter, all accesses will be billed at the public
upload/download rates, <irrespective of where the Hadoop cluster is>.
*** Example: HP cloud service definition
Here is an example that connects to the HP Cloud object store.
+--
<property>
<name>fs.swift.service.hpcloud.auth.url</name>
<value>https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
</value>
<description>HP Cloud</description>
</property>
<property>
<name>fs.swift.service.hpcloud.tenant</name>
<value>FE806AA86</value>
</property>
<property>
<name>fs.swift.service.hpcloud.username</name>
<value>FE806AA86DFFFE2F6ED8</value>
</property>
<property>
<name>fs.swift.service.hpcloud.password</name>
<value>secret-password-goes-here</value>
</property>
<property>
<name>fs.swift.service.hpcloud.public</name>
<value>true</value>
</property>
+--
A reference to this service would use the <<<hpcloud>>> service name:
+--
swift://hadoop-container.hpcloud/
+--
** General Swift Filesystem configuration options
Some configuration options apply to the Swift client, independent of
the specific Swift filesystem chosen.
*** Blocksize fs.swift.blocksize
Swift does not break up files into blocks, except in the special case of files
over 5GB in length. Accordingly, there isn't a notion of a "block size"
to define where the data is kept.
Hadoop's MapReduce layer depends on files declaring their block size,
so that it knows how to partition work. Too small a blocksize means that
many mappers work on small pieces of data; too large a block size means
that only a few mappers get started.
The block size value reported by Swift, therefore, controls the basic workload
partioning of the MapReduce engine -and can be an important parameter to
tune for performance of the cluster.
The property has a unit of kilobytes; the default value is <<<32*1024>>>: 32 MB
+--
<property>
<name>fs.swift.blocksize</name>
<value>32768</value>
</property>
+--
This blocksize has no influence on how files are stored in Swift; it only controls
what the reported size of blocks are - a value used in Hadoop MapReduce to
divide work.
Note that the MapReduce engine's split logic can be tuned independently by setting
the <<<mapred.min.split.size>>> and <<<mapred.max.split.size>>> properties,
which can be done in specific job configurations.
+--
<property>
<name>mapred.min.split.size</name>
<value>524288</value>
</property>
<property>
<name>mapred.max.split.size</name>
<value>1048576</value>
</property>
+--
In an Apache Pig script, these properties would be set as:
---
mapred.min.split.size 524288
mapred.max.split.size 1048576
---
*** Partition size fs.swift.partsize
The Swift filesystem client breaks very large files into partitioned files,
uploading each as it progresses, and writing any remaning data and an XML
manifest when a partitioned file is closed.
The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that
Swift can support.
It is possible to set a smaller partition size, in the <<<fs.swift.partsize>>>
option. This takes a value in KB.
+--
<property>
<name>fs.swift.partsize</name>
<value>1024</value>
<description>upload every MB</description>
</property>
+--
When should this value be changed from its default?
While there is no need to ever change it for basic operation of
the Swift filesystem client, it can be tuned
* If a Swift filesystem is location aware, then breaking a file up into
smaller partitions scatters the data round the cluster. For best performance,
the property <<<fs.swift.blocksize>>> should be set to a smaller value than the
partition size of files.
* When writing to an unpartitioned file, the entire write is done in the
<<<close()>>> operation. When a file is partitioned, the outstanding data to
be written whenever the outstanding amount of data is greater than the
partition size. This means that data will be written more incrementally
*** Request size fs.swift.requestsize
The Swift filesystem client reads files in HTTP GET operations, asking for
a block of data at a time.
The default value is 64KB. A larger value may be more efficient over faster
networks, as it reduces the overhead of setting up the HTTP operation.
However, if the file is read with many random accesses, requests for
data will be made from different parts of the file -discarding some of the
previously requested data. The benefits of larger request sizes may be wasted.
The property <<<fs.swift.requestsize>>> sets the request size in KB.
+--
<property>
<name>fs.swift.requestsize</name>
<value>128</value>
</property>
+--
*** Connection timeout fs.swift.connect.timeout
This sets the timeout in milliseconds to connect to a Swift service.
+--
<property>
<name>fs.swift.connect.timeout</name>
<value>15000</value>
</property>
+--
A shorter timeout means that connection failures are raised faster -but
may trigger more false alarms. A longer timeout is more resilient to network
problems -and may be needed when talking to remote filesystems.
*** Connection timeout fs.swift.socket.timeout
This sets the timeout in milliseconds to wait for data from a connected socket.
+--
<property>
<name>fs.swift.socket.timeout</name>
<value>60000</value>
</property>
+--
A shorter timeout means that connection failures are raised faster -but
may trigger more false alarms. A longer timeout is more resilient to network
problems -and may be needed when talking to remote filesystems.
*** Connection Retry Count fs.swift.connect.retry.count
This sets the number of times to try to connect to a service whenever
an HTTP request is made.
+--
<property>
<name>fs.swift.connect.retry.count</name>
<value>3</value>
</property>
+--
The more retries, the more resilient it is to transient outages -and the
less rapid it is at detecting and reporting server connectivity problems.
*** Connection Throttle Delay fs.swift.connect.throttle.delay
This property adds a delay between bulk file copy and delete operations,
to prevent requests being throttled or blocked by the remote service
+--
<property>
<name>fs.swift.connect.throttle.delay</name>
<value>0</value>
</property>
+--
It is measured in milliseconds; "0" means do not add any delay.
Throttling is enabled on the public endpoints of some Swift services.
If <<<rename()>>> or <<<delete()>>> operations fail with
<<<SwiftThrottledRequestException>>>
exceptions, try setting this property.
*** HTTP Proxy
If the client can only access the Swift filesystem via a web proxy
server, the client configuration must specify the proxy via
the <<<fs.swift.connect.proxy.host>>> and <<<fs.swift.connect.proxy.port>>>
properties.
+--
<property>
<name>fs.swift.proxy.host</name>
<value>web-proxy</value>
</property>
<property>
<name>fs.swift.proxy.port</name>
<value>8088</value>
</property>
+--
If the host is declared, the proxy port must be set to a valid integer value.
** Troubleshooting
*** ClassNotFoundException
The <<<hadoop-openstack>>> JAR -or any dependencies- may not be on your classpath.
If it is a remote MapReduce job that is failing, make sure that the JAR is
installed on the servers in the cluster -or that the job submission process
uploads the JAR file to the distributed cache.
*** Failure to Authenticate
A <<<SwiftAuthenticationFailedException>>> is thrown when the client
cannot authenticate with the OpenStack keystone server. This could be
because the URL in the service definition is wrong, or because
the supplied credentials are invalid.
[[1]] Check the authentication URL through <<<curl>>> or your browser
[[1]] Use a Swift client such as CyberDuck to validate your credentials
[[1]] If you have included a tenant ID, try leaving it out. Similarly,
try adding it if you had not included it.
[[1]] Try switching from API key authentication to password-based authentication,
by setting the password.
[[1]] Change your credentials. As with Amazon AWS clients, some credentials
don't seem to like going over the network.
*** Timeout connecting to the Swift Service
This happens if the client application is running outside an OpenStack cluster,
where it does not have access to the private hostname/IP address for filesystem
operations. Set the <<<public>>> flag to true -but remember to set it to false
for use in-cluster.
** Warnings
[[1]] Do not share your login details with anyone, which means do not log the
details, or check the XML configuration files into any revision control system
to which you do not have exclusive access.
[[1]] Similarly, do not use your real account details in any documentation *or any
bug reports submitted online*
[[1]] Prefer the apikey authentication over passwords as it is easier
to revoke a key -and some service providers allow you to set
an automatic expiry date on a key when issued.
[[1]] Do not use the public service endpoint from within a public OpenStack
cluster, as it will run up large bills.
[[1]] Remember: it's not a real filesystem or hierarchical directory structure.
Some operations (directory rename and delete) take time and are not atomic or
isolated from other operations taking place.
[[1]] Append is not supported.
[[1]] Unix-style permissions are not supported. All accounts with write access to
a repository have unlimited access; the same goes for those with read access.
[[1]] In the public clouds, do not make the containers public unless you are happy
with anyone reading your data, and are prepared to pay the costs of their
downloads.
** Limits
* Maximum length of an object path: 1024 characters
* Maximum size of a binary object: no absolute limit. Files > 5GB are
partitioned into separate files in the native filesystem, and merged during
retrieval. <Warning:> the partitioned/large file support is the
most complex part of the Hadoop/Swift FS integration, and, along with
authentication, the most troublesome to support.
** Testing the hadoop-openstack module
The <<<hadoop-openstack>>> can be remotely tested against any public
or private cloud infrastructure which supports the OpenStack Keystone
authentication mechanism. It can also be tested against private
OpenStack clusters. OpenStack Development teams are strongly encouraged to test
the Hadoop swift filesystem client against any version of Swift that they
are developing or deploying, to stress their cluster and to identify
bugs early.
The module comes with a large suite of JUnit tests -tests that are
only executed if the source tree includes credentials to test against a
specific cluster.
After checking out the Hadoop source tree, create the file:
+--
hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
+--
Into this file, insert the credentials needed to bond to the test filesystem,
as decribed above.
Next set the property <<<test.fs.swift.name>>> to the URL of a
swift container to test against. The tests expect exclusive access
to this container -do not keep any other data on it, or expect it
to be preserved.
+--
<property>
<name>test.fs.swift.name</name>
<value>swift://test.myswift/</value>
</property>
+--
In the base hadoop directory, run:
+--
mvn clean install -DskipTests
+--
This builds a set of Hadoop JARs consistent with the <<<hadoop-openstack>>>
module that is about to be tested.
In the <<<hadoop-tools/hadoop-openstack>>> directory run
+--
mvn test -Dtest=TestSwiftRestClient
+--
This runs some simple tests which include authenticating
against the remote swift service. If these tests fail, so will all
the rest. If it does fail: check your authentication.
Once this test succeeds, you can run the full test suite
+--
mvn test
+--
Be advised that these tests can take an hour or more, especially against a
remote Swift service -or one that throttles bulk operations.
Once the <<<auth-keys.xml>>> file is in place, the <<<mvn test>>> runs from
the Hadoop source base directory will automatically run these OpenStack tests
While this ensures that no regressions have occurred, it can also add significant
time to test runs, and may run up bills, depending on who is providing\
the Swift storage service. We recommend having a separate source tree
set up purely for the Swift tests, and running it manually or by the CI tooling
at a lower frequency than normal test runs.
Finally: Apache Hadoop is an open source project. Contributions of code
-including more tests- are very welcome.

46
hadoop-tools/hadoop-openstack/src/site/site.xml Normal file
View File

@ -0,0 +1,46 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project name="Apache Hadoop ${project.version}">
<skin>
<groupId>org.apache.maven.skins</groupId>
<artifactId>maven-stylus-skin</artifactId>
<version>1.2</version>
</skin>
<body>
<links>
<item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
</links>
</body>
</project>

31
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
/**
* A path filter that accepts everything
*/
public class AcceptAllFilter implements PathFilter {
@Override
public boolean accept(Path file) {
return true;
}
}

400
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java Normal file
View File

@ -0,0 +1,400 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystemStore;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanupInTeardown;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.getServiceURI;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.noteAction;
/**
* This is the base class for most of the Swift tests
*/
public class SwiftFileSystemBaseTest extends Assert implements
SwiftTestConstants {
protected static final Log LOG =
LogFactory.getLog(SwiftFileSystemBaseTest.class);
protected SwiftNativeFileSystem fs;
protected static SwiftNativeFileSystem lastFs;
protected byte[] data = SwiftTestUtils.dataset(getBlockSize() * 2, 0, 255);
private Configuration conf;
@Before
public void setUp() throws Exception {
noteAction("setup");
final URI uri = getFilesystemURI();
conf = createConfiguration();
fs = createSwiftFS();
try {
fs.initialize(uri, conf);
} catch (IOException e) {
//FS init failed, set it to null so that teardown doesn't
//attempt to use it
fs = null;
throw e;
}
//remember the last FS
lastFs = fs;
noteAction("setup complete");
}
/**
* Configuration generator. May be overridden to inject
* some custom options
* @return a configuration with which to create FS instances
*/
protected Configuration createConfiguration() {
return new Configuration();
}
@After
public void tearDown() throws Exception {
cleanupInTeardown(fs, "/test");
}
@AfterClass
public static void classTearDown() throws Exception {
if (lastFs != null) {
List<DurationStats> statistics = lastFs.getOperationStatistics();
for (DurationStats stat : statistics) {
LOG.info(stat.toString());
}
}
}
/**
* Get the configuration used to set up the FS
* @return the configuration
*/
public Configuration getConf() {
return conf;
}
/**
* Describe the test, combining some logging with details
* for people reading the code
*
* @param description test description
*/
protected void describe(String description) {
noteAction(description);
}
protected URI getFilesystemURI() throws URISyntaxException, IOException {
return getServiceURI(createConfiguration());
}
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
SwiftNativeFileSystem swiftNativeFileSystem =
new SwiftNativeFileSystem();
return swiftNativeFileSystem;
}
protected int getBlockSize() {
return 1024;
}
/**
* Is rename supported?
* @return true
*/
protected boolean renameSupported() {
return true;
}
/**
* assume in a test that rename is supported;
* skip it if not
*/
protected void assumeRenameSupported() {
Assume.assumeTrue(renameSupported());
}
/**
* Take an unqualified path, and qualify it w.r.t the
* current filesystem
* @param pathString source path
* @return a qualified path instance
*/
protected Path path(String pathString) {
return new Path(pathString).makeQualified(fs);
}
/**
* Get the filesystem
* @return the current FS
*/
public SwiftNativeFileSystem getFs() {
return fs;
}
/**
* Create a file using the standard {@link #data} bytes.
*
* @param path path to write
* @throws IOException on any problem
*/
protected void createFile(Path path) throws IOException {
createFile(path, data);
}
/**
* Create a file with the given data.
*
* @param path path to write
* @param sourceData source dataset
* @throws IOException on any problem
*/
protected void createFile(Path path, byte[] sourceData) throws IOException {
FSDataOutputStream out = fs.create(path);
out.write(sourceData, 0, sourceData.length);
out.close();
}
/**
* Create and then close a file
* @param path path to create
* @throws IOException on a failure
*/
protected void createEmptyFile(Path path) throws IOException {
FSDataOutputStream out = fs.create(path);
out.close();
}
/**
* Get the inner store -useful for lower level operations
*
* @return the store
*/
protected SwiftNativeFileSystemStore getStore() {
return fs.getStore();
}
/**
* Rename a path
* @param src source
* @param dst dest
* @param renameMustSucceed flag to say "this rename must exist"
* @param srcExists add assert that the source exists afterwards
* @param dstExists add assert the dest exists afterwards
* @throws IOException IO trouble
*/
protected void rename(Path src, Path dst, boolean renameMustSucceed,
boolean srcExists, boolean dstExists) throws IOException {
if (renameMustSucceed) {
renameToSuccess(src, dst, srcExists, dstExists);
} else {
renameToFailure(src, dst);
}
}
/**
* Get a string describing the outcome of a rename, by listing the dest
* path and its parent along with some covering text
* @param src source patj
* @param dst dest path
* @return a string for logs and exceptions
* @throws IOException IO problems
*/
private String getRenameOutcome(Path src, Path dst) throws IOException {
String lsDst = ls(dst);
Path parent = dst.getParent();
String lsParent = parent != null ? ls(parent) : "";
return " result of " + src + " => " + dst
+ " - " + lsDst
+ " \n" + lsParent;
}
/**
* Rename, expecting an exception to be thrown
*
* @param src source
* @param dst dest
* @throws IOException a failure other than an
* expected SwiftRenameException or FileNotFoundException
*/
protected void renameToFailure(Path src, Path dst) throws IOException {
try {
getStore().rename(src, dst);
fail("Expected failure renaming " + src + " to " + dst
+ "- but got success");
} catch (SwiftOperationFailedException e) {
LOG.debug("Rename failed (expected):" + e);
} catch (FileNotFoundException e) {
LOG.debug("Rename failed (expected):" + e);
}
}
/**
* Rename to success
*
* @param src source
* @param dst dest
* @param srcExists add assert that the source exists afterwards
* @param dstExists add assert the dest exists afterwards
* @throws SwiftOperationFailedException operation failure
* @throws IOException IO problems
*/
protected void renameToSuccess(Path src, Path dst,
boolean srcExists, boolean dstExists)
throws SwiftOperationFailedException, IOException {
getStore().rename(src, dst);
String outcome = getRenameOutcome(src, dst);
assertEquals("Source " + src + "exists: " + outcome,
srcExists, fs.exists(src));
assertEquals("Destination " + dstExists + " exists" + outcome,
dstExists, fs.exists(dst));
}
/**
* List a path in the test FS
* @param path path to list
* @return the contents of the path/dir
* @throws IOException IO problems
*/
protected String ls(Path path) throws IOException {
return SwiftTestUtils.ls(fs, path);
}
/**
* assert that a path exists
* @param message message to use in an assertion
* @param path path to probe
* @throws IOException IO problems
*/
public void assertExists(String message, Path path) throws IOException {
assertPathExists(fs, message, path);
}
/**
* assert that a path does not
* @param message message to use in an assertion
* @param path path to probe
* @throws IOException IO problems
*/
public void assertPathDoesNotExist(String message, Path path) throws
IOException {
SwiftTestUtils.assertPathDoesNotExist(fs, message, path);
}
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
*
* @param filename name of the file
* @throws IOException IO problems during file operations
*/
protected void assertIsFile(Path filename) throws IOException {
SwiftTestUtils.assertIsFile(fs, filename);
}
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
*
* @throws IOException IO problems during file operations
*/
protected void mkdirs(Path path) throws IOException {
assertTrue("Failed to mkdir" + path, fs.mkdirs(path));
}
/**
* Assert that a delete succeeded
* @param path path to delete
* @param recursive recursive flag
* @throws IOException IO problems
*/
protected void assertDeleted(Path path, boolean recursive) throws IOException {
SwiftTestUtils.assertDeleted(fs, path, recursive);
}
/**
* Assert that a value is not equal to the expected value
* @param message message if the two values are equal
* @param expected expected value
* @param actual actual value
*/
protected void assertNotEqual(String message, int expected, int actual) {
assertTrue(message,
actual != expected);
}
/**
* Get the number of partitions written from the Swift Native FS APIs
* @param out output stream
* @return the number of partitioned files written by the stream
*/
protected int getPartitionsWritten(FSDataOutputStream out) {
return SwiftNativeFileSystem.getPartitionsWritten(out);
}
/**
* Assert that the no. of partitions written matches expectations
* @param action operation (for use in the assertions)
* @param out output stream
* @param expected expected no. of partitions
*/
protected void assertPartitionsWritten(String action, FSDataOutputStream out,
long expected) {
OutputStream nativeStream = out.getWrappedStream();
int written = getPartitionsWritten(out);
if(written !=expected) {
Assert.fail(action + ": " +
TestSwiftFileSystemPartitionedUploads.WRONG_PARTITION_COUNT
+ " + expected: " + expected + " actual: " + written
+ " -- " + nativeStream);
}
}
/**
* Assert that the result value == -1; which implies
* that a read was successful
* @param text text to include in a message (usually the operation)
* @param result read result to validate
*/
protected void assertMinusOne(String text, int result) {
assertEquals(text + " wrong read result " + result, -1, result);
}
}

34
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
/**
* Hard coded constants for the test timeouts
*/
public interface SwiftTestConstants {
/**
* Timeout for swift tests: {@value}
*/
int SWIFT_TEST_TIMEOUT = 5 * 60 * 1000;
/**
* Timeout for tests performing bulk operations: {@value}
*/
int SWIFT_BULK_IO_TEST_TIMEOUT = 12 * 60 * 1000;
}

366
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java Normal file
View File

@ -0,0 +1,366 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSMainOperationsBaseTest;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import static org.apache.hadoop.fs.swift.SwiftTestConstants.SWIFT_TEST_TIMEOUT;
import java.io.IOException;
import java.net.URI;
public class TestFSMainOperationsSwift extends FSMainOperationsBaseTest {
@Override
@Before
public void setUp() throws Exception {
Configuration conf = new Configuration();
//small blocksize for faster remote tests
conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, 2);
URI serviceURI = SwiftTestUtils.getServiceURI(conf);
fSys = FileSystem.get(serviceURI, conf);
super.setUp();
}
private Path wd = null;
@Override
protected FileSystem createFileSystem() throws Exception {
return fSys;
}
@Override
protected Path getDefaultWorkingDirectory() throws IOException {
if (wd == null) {
wd = fSys.getWorkingDirectory();
}
return wd;
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWDAbsolute() throws IOException {
Path absoluteDir = getTestRootPath(fSys, "test/existingDir");
fSys.mkdirs(absoluteDir);
fSys.setWorkingDirectory(absoluteDir);
Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testListStatusThrowsExceptionForUnreadableDir() {
SwiftTestUtils.skip("unsupported");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testFsStatus() throws Exception {
super.testFsStatus();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWorkingDirectory() throws Exception {
super.testWorkingDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testMkdirs() throws Exception {
super.testMkdirs();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
super.testMkdirsFailsForSubdirectoryOfExistingFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGetFileStatusThrowsExceptionForNonExistentFile() throws
Exception {
super.testGetFileStatusThrowsExceptionForNonExistentFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testListStatusThrowsExceptionForNonExistentFile() throws
Exception {
super.testListStatusThrowsExceptionForNonExistentFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testListStatus() throws Exception {
super.testListStatus();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testListStatusFilterWithNoMatches() throws Exception {
super.testListStatusFilterWithNoMatches();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testListStatusFilterWithSomeMatches() throws Exception {
super.testListStatusFilterWithSomeMatches();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusNonExistentFile() throws Exception {
super.testGlobStatusNonExistentFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusWithNoMatchesInPath() throws Exception {
super.testGlobStatusWithNoMatchesInPath();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusSomeMatchesInDirectories() throws Exception {
super.testGlobStatusSomeMatchesInDirectories();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusWithMultipleWildCardMatches() throws Exception {
super.testGlobStatusWithMultipleWildCardMatches();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusWithMultipleMatchesOfSingleChar() throws Exception {
super.testGlobStatusWithMultipleMatchesOfSingleChar();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusFilterWithEmptyPathResults() throws Exception {
super.testGlobStatusFilterWithEmptyPathResults();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusFilterWithSomePathMatchesAndTrivialFilter() throws
Exception {
super.testGlobStatusFilterWithSomePathMatchesAndTrivialFilter();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter() throws
Exception {
super.testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter() throws
Exception {
super.testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter() throws
Exception {
super.testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter() throws
Exception {
super.testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWriteReadAndDeleteEmptyFile() throws Exception {
super.testWriteReadAndDeleteEmptyFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWriteReadAndDeleteHalfABlock() throws Exception {
super.testWriteReadAndDeleteHalfABlock();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWriteReadAndDeleteOneBlock() throws Exception {
super.testWriteReadAndDeleteOneBlock();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception {
super.testWriteReadAndDeleteOneAndAHalfBlocks();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWriteReadAndDeleteTwoBlocks() throws Exception {
super.testWriteReadAndDeleteTwoBlocks();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testOverwrite() throws IOException {
super.testOverwrite();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testWriteInNonExistentDirectory() throws IOException {
super.testWriteInNonExistentDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testDeleteNonExistentFile() throws IOException {
super.testDeleteNonExistentFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testDeleteRecursively() throws IOException {
super.testDeleteRecursively();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testDeleteEmptyDirectory() throws IOException {
super.testDeleteEmptyDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameNonExistentPath() throws Exception {
super.testRenameNonExistentPath();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameFileToNonExistentDirectory() throws Exception {
super.testRenameFileToNonExistentDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameFileToDestinationWithParentFile() throws Exception {
super.testRenameFileToDestinationWithParentFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameFileToExistingParent() throws Exception {
super.testRenameFileToExistingParent();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameFileToItself() throws Exception {
super.testRenameFileToItself();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameFileAsExistingFile() throws Exception {
super.testRenameFileAsExistingFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameFileAsExistingDirectory() throws Exception {
super.testRenameFileAsExistingDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameDirectoryToItself() throws Exception {
super.testRenameDirectoryToItself();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameDirectoryToNonExistentParent() throws Exception {
super.testRenameDirectoryToNonExistentParent();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameDirectoryAsNonExistentDirectory() throws Exception {
super.testRenameDirectoryAsNonExistentDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameDirectoryAsEmptyDirectory() throws Exception {
super.testRenameDirectoryAsEmptyDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameDirectoryAsNonEmptyDirectory() throws Exception {
super.testRenameDirectoryAsNonEmptyDirectory();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testRenameDirectoryAsFile() throws Exception {
super.testRenameDirectoryAsFile();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testInputStreamClosedTwice() throws IOException {
super.testInputStreamClosedTwice();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testOutputStreamClosedTwice() throws IOException {
super.testOutputStreamClosedTwice();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testGetWrappedInputStream() throws IOException {
super.testGetWrappedInputStream();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
@Override
public void testCopyToLocalWithUseRawLocalFileSystemOption() throws
Exception {
super.testCopyToLocalWithUseRawLocalFileSystemOption();
}
}

63
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.Test;
import java.net.URL;
/**
* This test just debugs which log resources are being picked up
*/
public class TestLogResources implements SwiftTestConstants {
protected static final Log LOG =
LogFactory.getLog(TestLogResources.class);
private void printf(String format, Object... args) {
String msg = String.format(format, args);
System.out.printf(msg + "\n");
LOG.info(msg);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testWhichLog4JPropsFile() throws Throwable {
locateResource("log4j.properties");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testWhichLog4JXMLFile() throws Throwable {
locateResource("log4j.XML");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCommonsLoggingProps() throws Throwable {
locateResource("commons-logging.properties");
}
private void locateResource(String resource) {
URL url = this.getClass().getClassLoader().getResource(resource);
if (url != null) {
printf("resource %s is at %s", resource, url);
} else {
printf("resource %s is not on the classpath", resource);
}
}
}

163
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java Normal file
View File

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Test;
/**
* Seek tests verify that
* <ol>
* <li>When you seek on a 0 byte file to byte (0), it's not an error.</li>
* <li>When you seek past the end of a file, it's an error that should
* raise -what- EOFException?</li>
* <li>when you seek forwards, you get new data</li>
* <li>when you seek backwards, you get the previous data</li>
* <li>That this works for big multi-MB files as well as small ones.</li>
* </ol>
* These may seem "obvious", but the more the input streams try to be clever
* about offsets and buffering, the more likely it is that seek() will start
* to get confused.
*/
public class TestReadPastBuffer extends SwiftFileSystemBaseTest {
protected static final Log LOG =
LogFactory.getLog(TestReadPastBuffer.class);
public static final int SWIFT_READ_BLOCKSIZE = 4096;
public static final int SEEK_FILE_LEN = SWIFT_READ_BLOCKSIZE * 2;
private Path testPath;
private Path readFile;
private Path zeroByteFile;
private FSDataInputStream instream;
/**
* Get a configuration which a small blocksize reported to callers
* @return a configuration for this test
*/
@Override
public Configuration getConf() {
Configuration conf = super.getConf();
/*
* set to 4KB
*/
conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, SWIFT_READ_BLOCKSIZE);
return conf;
}
/**
* Setup creates dirs under test/hadoop
*
* @throws Exception
*/
@Override
public void setUp() throws Exception {
super.setUp();
byte[] block = SwiftTestUtils.dataset(SEEK_FILE_LEN, 0, 255);
//delete the test directory
testPath = path("/test");
readFile = new Path(testPath, "TestReadPastBuffer.txt");
createFile(readFile, block);
}
@After
public void cleanFile() {
IOUtils.closeStream(instream);
instream = null;
}
/**
* Create a config with a 1KB request size
* @return a config
*/
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
return conf;
}
/**
* Seek past the buffer then read
* @throws Throwable problems
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekAndReadPastEndOfFile() throws Throwable {
instream = fs.open(readFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
//go just before the end
instream.seek(SEEK_FILE_LEN - 2);
assertTrue("Premature EOF", instream.read() != -1);
assertTrue("Premature EOF", instream.read() != -1);
assertMinusOne("read past end of file", instream.read());
}
/**
* Seek past the buffer and attempt a read(buffer)
* @throws Throwable failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekBulkReadPastEndOfFile() throws Throwable {
instream = fs.open(readFile);
assertEquals(0, instream.getPos());
//go just before the end
instream.seek(SEEK_FILE_LEN - 1);
byte[] buffer = new byte[1];
int result = instream.read(buffer, 0, 1);
//next byte is expected to fail
result = instream.read(buffer, 0, 1);
assertMinusOne("read past end of file", result);
//and this one
result = instream.read(buffer, 0, 1);
assertMinusOne("read past end of file", result);
//now do an 0-byte read and expect it to
//to be checked first
result = instream.read(buffer, 0, 0);
assertEquals("EOF checks coming before read range check", 0, result);
}
/**
* Read past the buffer size byte by byte and verify that it refreshed
* @throws Throwable
*/
@Test
public void testReadPastBufferSize() throws Throwable {
instream = fs.open(readFile);
while (instream.read() != -1);
//here we have gone past the end of a file and its buffer. Now try again
assertMinusOne("reading after the (large) file was read: "+ instream,
instream.read());
}
}

260
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java Normal file
View File

@ -0,0 +1,260 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Test;
import java.io.EOFException;
import java.io.IOException;
/**
* Seek tests verify that
* <ol>
* <li>When you seek on a 0 byte file to byte (0), it's not an error.</li>
* <li>When you seek past the end of a file, it's an error that should
* raise -what- EOFException?</li>
* <li>when you seek forwards, you get new data</li>
* <li>when you seek backwards, you get the previous data</li>
* <li>That this works for big multi-MB files as well as small ones.</li>
* </ol>
* These may seem "obvious", but the more the input streams try to be clever
* about offsets and buffering, the more likely it is that seek() will start
* to get confused.
*/
public class TestSeek extends SwiftFileSystemBaseTest {
protected static final Log LOG =
LogFactory.getLog(TestSeek.class);
public static final int SMALL_SEEK_FILE_LEN = 256;
private Path testPath;
private Path smallSeekFile;
private Path zeroByteFile;
private FSDataInputStream instream;
/**
* Setup creates dirs under test/hadoop
*
* @throws Exception
*/
@Override
public void setUp() throws Exception {
super.setUp();
//delete the test directory
testPath = path("/test");
smallSeekFile = new Path(testPath, "seekfile.txt");
zeroByteFile = new Path(testPath, "zero.txt");
byte[] block = SwiftTestUtils.dataset(SMALL_SEEK_FILE_LEN, 0, 255);
//this file now has a simple rule: offset => value
createFile(smallSeekFile, block);
createEmptyFile(zeroByteFile);
}
@After
public void cleanFile() {
IOUtils.closeStream(instream);
instream = null;
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekZeroByteFile() throws Throwable {
instream = fs.open(zeroByteFile);
assertEquals(0, instream.getPos());
//expect initial read to fai;
int result = instream.read();
assertMinusOne("initial byte read", result);
byte[] buffer = new byte[1];
//expect that seek to 0 works
instream.seek(0);
//reread, expect same exception
result = instream.read();
assertMinusOne("post-seek byte read", result);
result = instream.read(buffer, 0, 1);
assertMinusOne("post-seek buffer read", result);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testBlockReadZeroByteFile() throws Throwable {
instream = fs.open(zeroByteFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
byte[] buffer = new byte[1];
int result = instream.read(buffer, 0, 1);
assertMinusOne("block read zero byte file", result);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekReadClosedFile() throws Throwable {
instream = fs.open(smallSeekFile);
instream.close();
try {
instream.seek(0);
} catch (SwiftConnectionClosedException e) {
//expected a closed file
}
try {
instream.read();
} catch (IOException e) {
//expected a closed file
}
try {
byte[] buffer = new byte[1];
int result = instream.read(buffer, 0, 1);
} catch (IOException e) {
//expected a closed file
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testNegativeSeek() throws Throwable {
instream = fs.open(smallSeekFile);
assertEquals(0, instream.getPos());
try {
instream.seek(-1);
long p = instream.getPos();
LOG.warn("Seek to -1 returned a position of " + p);
int result = instream.read();
fail(
"expected an exception, got data " + result + " at a position of " + p);
} catch (IOException e) {
//bad seek -expected
}
assertEquals(0, instream.getPos());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekFile() throws Throwable {
instream = fs.open(smallSeekFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
instream.seek(0);
int result = instream.read();
assertEquals(0, result);
assertEquals(1, instream.read());
assertEquals(2, instream.getPos());
assertEquals(2, instream.read());
assertEquals(3, instream.getPos());
instream.seek(128);
assertEquals(128, instream.getPos());
assertEquals(128, instream.read());
instream.seek(63);
assertEquals(63, instream.read());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekAndReadPastEndOfFile() throws Throwable {
instream = fs.open(smallSeekFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
//go just before the end
instream.seek(SMALL_SEEK_FILE_LEN - 2);
assertTrue("Premature EOF", instream.read() != -1);
assertTrue("Premature EOF", instream.read() != -1);
assertMinusOne("read past end of file", instream.read());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekAndPastEndOfFileThenReseekAndRead() throws Throwable {
instream = fs.open(smallSeekFile);
//go just before the end. This may or may not fail; it may be delayed until the
//read
try {
instream.seek(SMALL_SEEK_FILE_LEN);
//if this doesn't trigger, then read() is expected to fail
assertMinusOne("read after seeking past EOF", instream.read());
} catch (EOFException expected) {
//here an exception was raised in seek
}
instream.seek(1);
assertTrue("Premature EOF", instream.read() != -1);
}
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
return conf;
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekBigFile() throws Throwable {
Path testSeekFile = new Path(testPath, "bigseekfile.txt");
byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
createFile(testSeekFile, block);
instream = fs.open(testSeekFile);
assertEquals(0, instream.getPos());
//expect that seek to 0 works
instream.seek(0);
int result = instream.read();
assertEquals(0, result);
assertEquals(1, instream.read());
assertEquals(2, instream.read());
//do seek 32KB ahead
instream.seek(32768);
assertEquals("@32768", block[32768], (byte) instream.read());
instream.seek(40000);
assertEquals("@40000", block[40000], (byte) instream.read());
instream.seek(8191);
assertEquals("@8191", block[8191], (byte) instream.read());
instream.seek(0);
assertEquals("@0", 0, (byte) instream.read());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
Path testSeekFile = new Path(testPath, "bigseekfile.txt");
byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
createFile(testSeekFile, block);
instream = fs.open(testSeekFile);
instream.seek(39999);
assertTrue(-1 != instream.read());
assertEquals (40000, instream.getPos());
byte[] readBuffer = new byte[256];
instream.read(128, readBuffer, 0, readBuffer.length);
//have gone back
assertEquals(40000, instream.getPos());
//content is the same too
assertEquals("@40000", block[40000], (byte) instream.read());
//now verify the picked up data
for (int i = 0; i < 256; i++) {
assertEquals("@" + i, block[i + 128], readBuffer[i]);
}
}
/**
* work out the expected byte from a specific offset
* @param offset offset in the file
* @return the value
*/
int expectedByte(int offset) {
return offset & 0xff;
}
}

194
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java Normal file
View File

@ -0,0 +1,194 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.swift.http.SwiftRestClient;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_LOCATION_AWARE;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_TENANT;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_BLOCKSIZE;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONNECTION_TIMEOUT;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PARTITION_SIZE;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_HOST_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_PORT_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_RETRY_COUNT;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PREFIX;
/**
* Test the swift service-specific configuration binding features
*/
public class TestSwiftConfig extends Assert {
public static final String SERVICE = "openstack";
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testEmptyUrl() throws Exception {
final Configuration configuration = new Configuration();
set(configuration, DOT_TENANT, "tenant");
set(configuration, DOT_USERNAME, "username");
set(configuration, DOT_PASSWORD, "password");
mkInstance(configuration);
}
@Test
public void testEmptyTenant() throws Exception {
final Configuration configuration = new Configuration();
set(configuration, DOT_AUTH_URL, "http://localhost:8080");
set(configuration, DOT_USERNAME, "username");
set(configuration, DOT_PASSWORD, "password");
mkInstance(configuration);
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testEmptyUsername() throws Exception {
final Configuration configuration = new Configuration();
set(configuration, DOT_AUTH_URL, "http://localhost:8080");
set(configuration, DOT_TENANT, "tenant");
set(configuration, DOT_PASSWORD, "password");
mkInstance(configuration);
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testEmptyPassword() throws Exception {
final Configuration configuration = new Configuration();
set(configuration, DOT_AUTH_URL, "http://localhost:8080");
set(configuration, DOT_TENANT, "tenant");
set(configuration, DOT_USERNAME, "username");
mkInstance(configuration);
}
@Test
public void testGoodRetryCount() throws Exception {
final Configuration configuration = createCoreConfig();
configuration.set(SWIFT_RETRY_COUNT, "3");
mkInstance(configuration);
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testBadRetryCount() throws Exception {
final Configuration configuration = createCoreConfig();
configuration.set(SWIFT_RETRY_COUNT, "three");
mkInstance(configuration);
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testBadConnectTimeout() throws Exception {
final Configuration configuration = createCoreConfig();
configuration.set(SWIFT_CONNECTION_TIMEOUT, "three");
mkInstance(configuration);
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testZeroBlocksize() throws Exception {
final Configuration configuration = createCoreConfig();
configuration.set(SWIFT_BLOCKSIZE, "0");
mkInstance(configuration);
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testNegativeBlocksize() throws Exception {
final Configuration configuration = createCoreConfig();
configuration.set(SWIFT_BLOCKSIZE, "-1");
mkInstance(configuration);
}
@Test
public void testPositiveBlocksize() throws Exception {
final Configuration configuration = createCoreConfig();
int size = 127;
configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
SwiftRestClient restClient = mkInstance(configuration);
assertEquals(size, restClient.getBlocksizeKB());
}
@Test
public void testLocationAwareTruePropagates() throws Exception {
final Configuration configuration = createCoreConfig();
set(configuration, DOT_LOCATION_AWARE, "true");
SwiftRestClient restClient = mkInstance(configuration);
assertTrue(restClient.isLocationAware());
}
@Test
public void testLocationAwareFalsePropagates() throws Exception {
final Configuration configuration = createCoreConfig();
set(configuration, DOT_LOCATION_AWARE, "false");
SwiftRestClient restClient = mkInstance(configuration);
assertFalse(restClient.isLocationAware());
}
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testNegativePartsize() throws Exception {
final Configuration configuration = createCoreConfig();
configuration.set(SWIFT_PARTITION_SIZE, "-1");
SwiftRestClient restClient = mkInstance(configuration);
}
@Test
public void testPositivePartsize() throws Exception {
final Configuration configuration = createCoreConfig();
int size = 127;
configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
SwiftRestClient restClient = mkInstance(configuration);
assertEquals(size, restClient.getPartSizeKB());
}
@Test
public void testProxyData() throws Exception {
final Configuration configuration = createCoreConfig();
String proxy="web-proxy";
int port = 8088;
configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
SwiftRestClient restClient = mkInstance(configuration);
assertEquals(proxy, restClient.getProxyHost());
assertEquals(port, restClient.getProxyPort());
}
private Configuration createCoreConfig() {
final Configuration configuration = new Configuration();
set(configuration, DOT_AUTH_URL, "http://localhost:8080");
set(configuration, DOT_TENANT, "tenant");
set(configuration, DOT_USERNAME, "username");
set(configuration, DOT_PASSWORD, "password");
return configuration;
}
private void set(Configuration configuration, String field, String value) {
configuration.set(SWIFT_SERVICE_PREFIX + SERVICE + field, value);
}
private SwiftRestClient mkInstance(Configuration configuration) throws
IOException,
URISyntaxException {
URI uri = new URI("swift://container.openstack/");
return SwiftRestClient.getInstance(uri, configuration);
}
}

289
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java Normal file
View File

@ -0,0 +1,289 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.IOException;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertFileHasLength;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertIsDirectory;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
/**
* Test basic filesystem operations.
* Many of these are similar to those in {@link TestSwiftFileSystemContract}
* -this is a JUnit4 test suite used to initially test the Swift
* component. Once written, there's no reason not to retain these tests.
*/
public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
private static final Log LOG =
LogFactory.getLog(TestSwiftFileSystemBasicOps.class);
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLsRoot() throws Throwable {
Path path = new Path("/");
FileStatus[] statuses = fs.listStatus(path);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testMkDir() throws Throwable {
Path path = new Path("/test/MkDir");
fs.mkdirs(path);
//success then -so try a recursive operation
fs.delete(path, true);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteNonexistentFile() throws Throwable {
Path path = new Path("/test/DeleteNonexistentFile");
assertFalse("delete returned true", fs.delete(path, false));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testPutFile() throws Throwable {
Path path = new Path("/test/PutFile");
Exception caught = null;
writeTextFile(fs, path, "Testing a put to a file", false);
assertDeleted(path, false);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testPutGetFile() throws Throwable {
Path path = new Path("/test/PutGetFile");
try {
String text = "Testing a put and get to a file "
+ System.currentTimeMillis();
writeTextFile(fs, path, text, false);
String result = readBytesToString(fs, path, text.length());
assertEquals(text, result);
} finally {
delete(fs, path);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testPutDeleteFileInSubdir() throws Throwable {
Path path =
new Path("/test/PutDeleteFileInSubdir/testPutDeleteFileInSubdir");
String text = "Testing a put and get to a file in a subdir "
+ System.currentTimeMillis();
writeTextFile(fs, path, text, false);
assertDeleted(path, false);
//now delete the parent that should have no children
assertDeleted(new Path("/test/PutDeleteFileInSubdir"), false);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRecursiveDelete() throws Throwable {
Path childpath =
new Path("/test/testRecursiveDelete");
String text = "Testing a put and get to a file in a subdir "
+ System.currentTimeMillis();
writeTextFile(fs, childpath, text, false);
//now delete the parent that should have no children
assertDeleted(new Path("/test"), true);
assertFalse("child entry still present " + childpath, fs.exists(childpath));
}
private void delete(SwiftNativeFileSystem fs, Path path) {
try {
if (!fs.delete(path, false)) {
LOG.warn("Failed to delete " + path);
}
} catch (IOException e) {
LOG.warn("deleting " + path, e);
}
}
private void deleteR(SwiftNativeFileSystem fs, Path path) {
try {
if (!fs.delete(path, true)) {
LOG.warn("Failed to delete " + path);
}
} catch (IOException e) {
LOG.warn("deleting " + path, e);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testOverwrite() throws Throwable {
Path path = new Path("/test/Overwrite");
try {
String text = "Testing a put to a file "
+ System.currentTimeMillis();
writeTextFile(fs, path, text, false);
assertFileHasLength(fs, path, text.length());
String text2 = "Overwriting a file "
+ System.currentTimeMillis();
writeTextFile(fs, path, text2, true);
assertFileHasLength(fs, path, text2.length());
String result = readBytesToString(fs, path, text2.length());
assertEquals(text2, result);
} finally {
delete(fs, path);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testOverwriteDirectory() throws Throwable {
Path path = new Path("/test/testOverwriteDirectory");
try {
fs.mkdirs(path.getParent());
String text = "Testing a put to a file "
+ System.currentTimeMillis();
writeTextFile(fs, path, text, false);
assertFileHasLength(fs, path, text.length());
} finally {
delete(fs, path);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testFileStatus() throws Throwable {
Path path = new Path("/test/FileStatus");
try {
String text = "Testing File Status "
+ System.currentTimeMillis();
writeTextFile(fs, path, text, false);
SwiftTestUtils.assertIsFile(fs, path);
} finally {
delete(fs, path);
}
}
/**
* Assert that a newly created directory is a directory
*
* @throws Throwable if not, or if something else failed
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDirStatus() throws Throwable {
Path path = new Path("/test/DirStatus");
try {
fs.mkdirs(path);
assertIsDirectory(fs, path);
} finally {
delete(fs, path);
}
}
/**
* Assert that if a directory that has children is deleted, it is still
* a directory
*
* @throws Throwable if not, or if something else failed
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDirStaysADir() throws Throwable {
Path path = new Path("/test/dirStaysADir");
Path child = new Path(path, "child");
try {
//create the dir
fs.mkdirs(path);
//assert the parent has the directory nature
assertIsDirectory(fs, path);
//create the child dir
writeTextFile(fs, child, "child file", true);
//assert the parent has the directory nature
assertIsDirectory(fs, path);
//now rm the child
delete(fs, child);
} finally {
deleteR(fs, path);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCreateMultilevelDir() throws Throwable {
Path base = new Path("/test/CreateMultilevelDir");
Path path = new Path(base, "1/2/3");
fs.mkdirs(path);
assertExists("deep multilevel dir not created", path);
fs.delete(base, true);
assertPathDoesNotExist("Multilevel delete failed", path);
assertPathDoesNotExist("Multilevel delete failed", base);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCreateDirWithFileParent() throws Throwable {
Path path = new Path("/test/CreateDirWithFileParent");
Path child = new Path(path, "subdir/child");
fs.mkdirs(path.getParent());
try {
//create the child dir
writeTextFile(fs, path, "parent", true);
try {
fs.mkdirs(child);
} catch (SwiftNotDirectoryException expected) {
LOG.debug("Expected Exception", expected);
}
} finally {
fs.delete(path, true);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLongObjectNamesForbidden() throws Throwable {
StringBuilder buffer = new StringBuilder(1200);
buffer.append("/");
for (int i = 0; i < (1200 / 4); i++) {
buffer.append(String.format("%04x", i));
}
String pathString = buffer.toString();
Path path = new Path(pathString);
try {
writeTextFile(fs, path, pathString, true);
//if we get here, problems.
fs.delete(path, false);
fail("Managed to create an object with a name of length "
+ pathString.length());
} catch (SwiftBadRequestException e) {
//expected
//LOG.debug("Caught exception " + e, e);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLsNonExistentFile() throws Exception {
try {
Path path = new Path("/test/hadoop/file");
FileStatus[] statuses = fs.listStatus(path);
fail("Should throw FileNotFoundException on " + path
+ " but got list of length " + statuses.length);
} catch (FileNotFoundException fnfe) {
// expected
}
}
}

167
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.IOException;
/**
* Test block location logic.
* The endpoint may or may not be location-aware
*/
public class TestSwiftFileSystemBlockLocation extends SwiftFileSystemBaseTest {
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateSingleFileBlocks() throws Throwable {
describe("verify that a file returns 1+ blocks");
FileStatus fileStatus = createFileAndGetStatus();
BlockLocation[] locations =
getFs().getFileBlockLocations(fileStatus, 0, 1);
assertNotEqual("No block locations supplied for " + fileStatus, 0,
locations.length);
for (BlockLocation location : locations) {
assertLocationValid(location);
}
}
private void assertLocationValid(BlockLocation location) throws
IOException {
LOG.info(location);
String[] hosts = location.getHosts();
String[] names = location.getNames();
assertNotEqual("No hosts supplied for " + location, 0, hosts.length);
//for every host, there's a name.
assertEquals("Unequal names and hosts in " + location,
hosts.length, names.length);
assertEquals(SwiftProtocolConstants.BLOCK_LOCATION,
location.getNames()[0]);
assertEquals(SwiftProtocolConstants.TOPOLOGY_PATH,
location.getTopologyPaths()[0]);
}
private FileStatus createFileAndGetStatus() throws IOException {
Path path = path("/test/locatedFile");
createFile(path);
return fs.getFileStatus(path);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateNullStatus() throws Throwable {
describe("verify that a null filestatus maps to a null location array");
BlockLocation[] locations =
getFs().getFileBlockLocations((FileStatus) null, 0, 1);
assertNull(locations);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateNegativeSeek() throws Throwable {
describe("verify that a negative offset is illegal");
try {
BlockLocation[] locations =
getFs().getFileBlockLocations(createFileAndGetStatus(),
-1,
1);
fail("Expected an exception, got " + locations.length + " locations");
} catch (IllegalArgumentException e) {
//expected
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateNegativeLen() throws Throwable {
describe("verify that a negative length is illegal");
try {
BlockLocation[] locations =
getFs().getFileBlockLocations(createFileAndGetStatus(),
0,
-1);
fail("Expected an exception, got " + locations.length + " locations");
} catch (IllegalArgumentException e) {
//expected
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateOutOfRangeLen() throws Throwable {
describe("overshooting the length is legal, as long as the" +
" origin location is valid");
BlockLocation[] locations =
getFs().getFileBlockLocations(createFileAndGetStatus(),
0,
data.length + 100);
assertNotNull(locations);
assertTrue(locations.length > 0);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateOutOfRangeSrc() throws Throwable {
describe("Seeking out of the file length returns an empty array");
BlockLocation[] locations =
getFs().getFileBlockLocations(createFileAndGetStatus(),
data.length + 100,
1);
assertEmptyBlockLocations(locations);
}
private void assertEmptyBlockLocations(BlockLocation[] locations) {
assertNotNull(locations);
if (locations.length!=0) {
fail("non empty locations[] with first entry of " + locations[0]);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateDirectory() throws Throwable {
describe("verify that locating a directory is an error");
createFile(path("/test/filename"));
FileStatus status = fs.getFileStatus(path("/test"));
LOG.info("Filesystem is " + fs + "; target is " + status);
SwiftTestUtils.assertIsDirectory(status);
BlockLocation[] locations;
locations = getFs().getFileBlockLocations(status,
0,
1);
assertEmptyBlockLocations(locations);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateRootDirectory() throws Throwable {
describe("verify that locating the root directory is an error");
FileStatus status = fs.getFileStatus(path("/"));
SwiftTestUtils.assertIsDirectory(status);
BlockLocation[] locations;
locations = getFs().getFileBlockLocations(status,
0,
1);
assertEmptyBlockLocations(locations);
}
}

60
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
/**
* Tests that blocksize is never zero for a file, either in the FS default
* or the FileStatus value of a queried file
*/
public class TestSwiftFileSystemBlocksize extends SwiftFileSystemBaseTest {
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDefaultBlocksizeNonZero() throws Throwable {
assertTrue("Zero default blocksize", 0L != getFs().getDefaultBlockSize());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDefaultBlocksizeRootPathNonZero() throws Throwable {
assertTrue("Zero default blocksize",
0L != getFs().getDefaultBlockSize(new Path("/")));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDefaultBlocksizeOtherPathNonZero() throws Throwable {
assertTrue("Zero default blocksize",
0L != getFs().getDefaultBlockSize(new Path("/test")));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testBlocksizeNonZeroForFile() throws Throwable {
Path smallfile = new Path("/test/smallfile");
SwiftTestUtils.writeTextFile(fs, smallfile, "blocksize", true);
createFile(smallfile);
FileStatus status = getFs().getFileStatus(smallfile);
assertTrue("Zero blocksize in " + status,
status.getBlockSize() != 0L);
assertTrue("Zero replication in " + status,
status.getReplication() != 0L);
}
}

105
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* Test Swift FS concurrency logic. This isn't a very accurate test,
* because it is hard to consistently generate race conditions.
* Consider it "best effort"
*/
public class TestSwiftFileSystemConcurrency extends SwiftFileSystemBaseTest {
protected static final Log LOG =
LogFactory.getLog(TestSwiftFileSystemConcurrency.class);
private Exception thread1Ex, thread2Ex;
public static final String TEST_RACE_CONDITION_ON_DELETE_DIR =
"/test/testraceconditionondirdeletetest";
/**
* test on concurrent file system changes
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRaceConditionOnDirDeleteTest() throws Exception {
SwiftTestUtils.skip("Skipping unreliable test");
final String message = "message";
final Path fileToRead = new Path(
TEST_RACE_CONDITION_ON_DELETE_DIR +"/files/many-files/file");
final ExecutorService executorService = Executors.newFixedThreadPool(2);
fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR +"/file/test/file1"));
fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR + "/documents/doc1"));
fs.create(new Path(
TEST_RACE_CONDITION_ON_DELETE_DIR + "/pictures/picture"));
executorService.execute(new Runnable() {
@Override
public void run() {
try {
assertDeleted(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR), true);
} catch (IOException e) {
LOG.warn("deletion thread:" + e, e);
thread1Ex = e;
throw new RuntimeException(e);
}
}
});
executorService.execute(new Runnable() {
@Override
public void run() {
try {
final FSDataOutputStream outputStream = fs.create(fileToRead);
outputStream.write(message.getBytes());
outputStream.close();
} catch (IOException e) {
LOG.warn("writer thread:" + e, e);
thread2Ex = e;
throw new RuntimeException(e);
}
}
});
executorService.awaitTermination(1, TimeUnit.MINUTES);
if (thread1Ex != null) {
throw thread1Ex;
}
if (thread2Ex != null) {
throw thread2Ex;
}
try {
fs.open(fileToRead);
LOG.info("concurrency test failed to trigger a failure");
} catch (FileNotFoundException expected) {
}
}
}

128
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java Normal file
View File

@ -0,0 +1,128 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import junit.framework.AssertionFailedError;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
/**
* This is the full filesystem contract test -which requires the
* Default config set up to point to a filesystem.
*
* Some of the tests override the base class tests -these
* are where SwiftFS does not implement those features, or
* when the behavior of SwiftFS does not match the normal
* contract -which normally means that directories and equal files
* are being treated as equal.
*/
public class TestSwiftFileSystemContract
extends FileSystemContractBaseTest {
private static final Log LOG =
LogFactory.getLog(TestSwiftFileSystemContract.class);
@Override
protected void setUp() throws Exception {
final URI uri = getFilesystemURI();
final Configuration conf = new Configuration();
fs = createSwiftFS();
try {
fs.initialize(uri, conf);
} catch (IOException e) {
//FS init failed, set it to null so that teardown doesn't
//attempt to use it
fs = null;
throw e;
}
super.setUp();
}
protected URI getFilesystemURI() throws URISyntaxException, IOException {
return SwiftTestUtils.getServiceURI(new Configuration());
}
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
SwiftNativeFileSystem swiftNativeFileSystem =
new SwiftNativeFileSystem();
return swiftNativeFileSystem;
}
@Override
public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
Path testDir = path("/test/hadoop");
assertFalse(fs.exists(testDir));
assertTrue(fs.mkdirs(testDir));
assertTrue(fs.exists(testDir));
Path filepath = path("/test/hadoop/file");
SwiftTestUtils.writeTextFile(fs, filepath, "hello, world", false);
Path testSubDir = new Path(filepath, "subdir");
SwiftTestUtils.assertPathDoesNotExist(fs, "subdir before mkdir", testSubDir);
try {
fs.mkdirs(testSubDir);
fail("Should throw IOException.");
} catch (SwiftNotDirectoryException e) {
// expected
assertEquals(filepath,e.getPath());
}
//now verify that the subdir path does not exist
SwiftTestUtils.assertPathDoesNotExist(fs, "subdir after mkdir", testSubDir);
Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir");
try {
fs.mkdirs(testDeepSubDir);
fail("Should throw IOException.");
} catch (SwiftNotDirectoryException e) {
// expected
}
SwiftTestUtils.assertPathDoesNotExist(fs, "testDeepSubDir after mkdir",
testDeepSubDir);
}
@Override
public void testWriteReadAndDeleteEmptyFile() throws Exception {
try {
super.testWriteReadAndDeleteEmptyFile();
} catch (AssertionFailedError e) {
SwiftTestUtils.downgrade("empty files get mistaken for directories", e);
}
}
@Override
public void testMkdirsWithUmask() throws Exception {
//unsupported
}
public void testZeroByteFilesAreFiles() throws Exception {
// SwiftTestUtils.unsupported("testZeroByteFilesAreFiles");
}
}

90
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.IOException;
/**
* Test deletion operations
*/
public class TestSwiftFileSystemDelete extends SwiftFileSystemBaseTest {
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteEmptyFile() throws IOException {
final Path file = new Path("/test/testDeleteEmptyFile");
createEmptyFile(file);
SwiftTestUtils.noteAction("about to delete");
assertDeleted(file, true);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteEmptyFileTwice() throws IOException {
final Path file = new Path("/test/testDeleteEmptyFileTwice");
createEmptyFile(file);
assertDeleted(file, true);
SwiftTestUtils.noteAction("multiple creates, and deletes");
assertFalse("Delete returned true", fs.delete(file, false));
createEmptyFile(file);
assertDeleted(file, true);
assertFalse("Delete returned true", fs.delete(file, false));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteNonEmptyFile() throws IOException {
final Path file = new Path("/test/testDeleteNonEmptyFile");
createFile(file);
assertDeleted(file, true);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteNonEmptyFileTwice() throws IOException {
final Path file = new Path("/test/testDeleteNonEmptyFileTwice");
createFile(file);
assertDeleted(file, true);
assertFalse("Delete returned true", fs.delete(file, false));
createFile(file);
assertDeleted(file, true);
assertFalse("Delete returned true", fs.delete(file, false));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteTestDir() throws IOException {
final Path file = new Path("/test/");
fs.delete(file, true);
assertPathDoesNotExist("Test dir found", file);
}
/**
* Test recursive root directory deletion fails if there is an entry underneath
* @throws Throwable
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRmRootDirRecursiveIsForbidden() throws Throwable {
Path root = path("/");
Path testFile = path("/test");
createFile(testFile);
assertTrue("rm(/) returned false", fs.delete(root, true));
assertExists("Root dir is missing", root);
assertPathDoesNotExist("test file not deleted", testFile);
}
}

141
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java Normal file
View File

@ -0,0 +1,141 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.FileNotFoundException;
/**
* Test swift-specific directory logic.
* This class is HDFS-1 compatible; its designed to be subclases by something
* with HDFS2 extensions
*/
public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
/**
* Asserts that a zero byte file has a status of file and not
* directory or symlink
*
* @throws Exception on failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testZeroByteFilesAreDirectories() throws Exception {
Path src = path("/test/testZeroByteFilesAreFiles");
//create a zero byte file
SwiftTestUtils.touch(fs, src);
SwiftTestUtils.assertIsDirectory(fs, src);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testNoStatusForMissingDirectories() throws Throwable {
Path missing = path("/test/testNoStatusForMissingDirectories");
assertPathDoesNotExist("leftover?", missing);
try {
FileStatus[] statuses = fs.listStatus(missing);
//not expected
fail("Expected a FileNotFoundException, got the status " + statuses);
} catch (FileNotFoundException expected) {
//expected
}
}
/**
* test that a dir off root has a listStatus() call that
* works as expected. and that when a child is added. it changes
*
* @throws Exception on failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
Path test = path("/test");
fs.delete(test, true);
mkdirs(test);
assertExists("created test directory", test);
FileStatus[] statuses = fs.listStatus(test);
String statusString = statusToString(test.toString(), statuses);
assertEquals("Wrong number of elements in file status " + statusString, 0,
statuses.length);
Path src = path("/test/file");
//create a zero byte file
SwiftTestUtils.touch(fs, src);
//stat it
statuses = fs.listStatus(test);
statusString = statusToString(test.toString(), statuses);
assertEquals("Wrong number of elements in file status " + statusString, 1,
statuses.length);
SwiftFileStatus stat = (SwiftFileStatus) statuses[0];
assertTrue("isDir(): Not a directory: " + stat, stat.isDir());
extraStatusAssertions(stat);
}
/**
* test that a dir two levels down has a listStatus() call that
* works as expected.
*
* @throws Exception on failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDirectoriesLowerDownHaveMatchingFileStatus() throws Exception {
Path test = path("/test/testDirectoriesLowerDownHaveMatchingFileStatus");
fs.delete(test, true);
mkdirs(test);
assertExists("created test sub directory", test);
FileStatus[] statuses = fs.listStatus(test);
String statusString = statusToString(test.toString(), statuses);
assertEquals("Wrong number of elements in file status " + statusString,0,
statuses.length);
}
private String statusToString(String pathname,
FileStatus[] statuses) {
assertNotNull(statuses);
return SwiftTestUtils.dumpStats(pathname,statuses);
}
/**
* method for subclasses to add extra assertions
* @param stat status to look at
*/
protected void extraStatusAssertions(SwiftFileStatus stat) {
}
/**
* Asserts that a zero byte file has a status of file and not
* directory or symlink
*
* @throws Exception on failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testMultiByteFilesAreFiles() throws Exception {
Path src = path("/test/testMultiByteFilesAreFiles");
SwiftTestUtils.writeTextFile(fs, src, "testMultiByteFilesAreFiles", false);
assertIsFile(src);
FileStatus status = fs.getFileStatus(src);
assertFalse(status.isDir());
}
}

143
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java Normal file
View File

@ -0,0 +1,143 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.http.RestClientBindings;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.Locale;
public class TestSwiftFileSystemExtendedContract extends SwiftFileSystemBaseTest {
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testOpenNonExistingFile() throws IOException {
final Path p = new Path("/test/testOpenNonExistingFile");
//open it as a file, should get FileNotFoundException
try {
final FSDataInputStream in = fs.open(p);
in.close();
fail("didn't expect to get here");
} catch (FileNotFoundException fnfe) {
LOG.debug("Expected: " + fnfe, fnfe);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testFilesystemHasURI() throws Throwable {
assertNotNull(fs.getUri());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCreateFile() throws Exception {
final Path f = new Path("/test/testCreateFile");
final FSDataOutputStream fsDataOutputStream = fs.create(f);
fsDataOutputStream.close();
assertExists("created file", f);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testWriteReadFile() throws Exception {
final Path f = new Path("/test/test");
final FSDataOutputStream fsDataOutputStream = fs.create(f);
final String message = "Test string";
fsDataOutputStream.write(message.getBytes());
fsDataOutputStream.close();
assertExists("created file", f);
FSDataInputStream open = null;
try {
open = fs.open(f);
final byte[] bytes = new byte[512];
final int read = open.read(bytes);
final byte[] buffer = new byte[read];
System.arraycopy(bytes, 0, buffer, 0, read);
assertEquals(message, new String(buffer));
} finally {
fs.delete(f, false);
IOUtils.closeStream(open);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConfDefinesFilesystem() throws Throwable {
Configuration conf = new Configuration();
SwiftTestUtils.getServiceURI(conf);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConfIsValid() throws Throwable {
Configuration conf = new Configuration();
URI fsURI = SwiftTestUtils.getServiceURI(conf);
RestClientBindings.bind(fsURI, conf);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
String scheme = fs.getScheme();
assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
/**
* Assert that a filesystem is case sensitive.
* This is done by creating a mixed-case filename and asserting that
* its lower case version is not there.
*
* @throws Exception failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testFilesystemIsCaseSensitive() throws Exception {
String mixedCaseFilename = "/test/UPPER.TXT";
Path upper = path(mixedCaseFilename);
Path lower = path(mixedCaseFilename.toLowerCase(Locale.ENGLISH));
assertFalse("File exists" + upper, fs.exists(upper));
assertFalse("File exists" + lower, fs.exists(lower));
FSDataOutputStream out = fs.create(upper);
out.writeUTF("UPPER");
out.close();
FileStatus upperStatus = fs.getFileStatus(upper);
assertExists("Original upper case file" + upper, upper);
//verify the lower-case version of the filename doesn't exist
assertPathDoesNotExist("lower case file", lower);
//now overwrite the lower case version of the filename with a
//new version.
out = fs.create(lower);
out.writeUTF("l");
out.close();
assertExists("lower case file", lower);
//verifEy the length of the upper file hasn't changed
assertExists("Original upper case file " + upper, upper);
FileStatus newStatus = fs.getFileStatus(upper);
assertEquals("Expected status:" + upperStatus
+ " actual status " + newStatus,
upperStatus.getLen(),
newStatus.getLen());
}
}

169
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
import java.io.IOException;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertListStatusFinds;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dumpStats;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.touch;
/**
* Test the FileSystem#listStatus() operations
*/
public class TestSwiftFileSystemLsOperations extends SwiftFileSystemBaseTest {
private Path[] testDirs;
/**
* Setup creates dirs under test/hadoop
*
* @throws Exception
*/
@Override
public void setUp() throws Exception {
super.setUp();
//delete the test directory
Path test = path("/test");
fs.delete(test, true);
mkdirs(test);
}
/**
* Create subdirectories and files under test/ for those tests
* that want them. Doing so adds overhead to setup and teardown,
* so should only be done for those tests that need them.
* @throws IOException on an IO problem
*/
private void createTestSubdirs() throws IOException {
testDirs = new Path[]{
path("/test/hadoop/a"),
path("/test/hadoop/b"),
path("/test/hadoop/c/1"),
};
assertPathDoesNotExist("test directory setup", testDirs[0]);
for (Path path : testDirs) {
mkdirs(path);
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListLevelTest() throws Exception {
createTestSubdirs();
FileStatus[] paths = fs.listStatus(path("/test"));
assertEquals(dumpStats("/test", paths), 1, paths.length);
assertEquals(path("/test/hadoop"), paths[0].getPath());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListLevelTestHadoop() throws Exception {
createTestSubdirs();
FileStatus[] paths;
paths = fs.listStatus(path("/test/hadoop"));
String stats = dumpStats("/test/hadoop", paths);
assertEquals("Paths.length wrong in " + stats, 3, paths.length);
assertEquals("Path element[0] wrong: " + stats, path("/test/hadoop/a"),
paths[0].getPath());
assertEquals("Path element[1] wrong: " + stats, path("/test/hadoop/b"),
paths[1].getPath());
assertEquals("Path element[2] wrong: " + stats, path("/test/hadoop/c"),
paths[2].getPath());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListStatusEmptyDirectory() throws Exception {
createTestSubdirs();
FileStatus[] paths;
paths = fs.listStatus(path("/test/hadoop/a"));
assertEquals(dumpStats("/test/hadoop/a", paths), 0,
paths.length);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListStatusFile() throws Exception {
describe("Create a single file under /test;" +
" assert that listStatus(/test) finds it");
Path file = path("/test/filename");
createFile(file);
FileStatus[] pathStats = fs.listStatus(file);
assertEquals(dumpStats("/test/", pathStats),
1,
pathStats.length);
//and assert that the len of that ls'd path is the same as the original
FileStatus lsStat = pathStats[0];
assertEquals("Wrong file len in listing of " + lsStat,
data.length, lsStat.getLen());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListEmptyRoot() throws Throwable {
describe("Empty the root dir and verify that an LS / returns {}");
cleanup("testListEmptyRoot", fs, "/test");
cleanup("testListEmptyRoot", fs, "/user");
FileStatus[] fileStatuses = fs.listStatus(path("/"));
assertEquals("Non-empty root" + dumpStats("/", fileStatuses),
0,
fileStatuses.length);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListNonEmptyRoot() throws Throwable {
Path test = path("/test");
touch(fs, test);
FileStatus[] fileStatuses = fs.listStatus(path("/"));
String stats = dumpStats("/", fileStatuses);
assertEquals("Wrong #of root children" + stats, 1, fileStatuses.length);
FileStatus status = fileStatuses[0];
assertEquals("Wrong path value" + stats,test, status.getPath());
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListStatusRootDir() throws Throwable {
Path dir = path("/");
Path child = path("/test");
touch(fs, child);
assertListStatusFinds(fs, dir, child);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListStatusFiltered() throws Throwable {
Path dir = path("/");
Path child = path("/test");
touch(fs, child);
FileStatus[] stats = fs.listStatus(dir, new AcceptAllFilter());
boolean found = false;
StringBuilder builder = new StringBuilder();
for (FileStatus stat : stats) {
builder.append(stat.toString()).append('\n');
if (stat.getPath().equals(child)) {
found = true;
}
}
assertTrue("Path " + child
+ " not found in directory " + dir + ":" + builder,
found);
}
}

442
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java Normal file
View File

@ -0,0 +1,442 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.httpclient.Header;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import org.junit.internal.AssumptionViolatedException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
/**
* Test partitioned uploads.
* This is done by forcing a very small partition size and verifying that it
* is picked up.
*/
public class TestSwiftFileSystemPartitionedUploads extends
SwiftFileSystemBaseTest {
public static final String WRONG_PARTITION_COUNT =
"wrong number of partitions written into ";
public static final int PART_SIZE = 1;
public static final int PART_SIZE_BYTES = PART_SIZE * 1024;
public static final int BLOCK_SIZE = 1024;
private URI uri;
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
//set the partition size to 1 KB
conf.setInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE, PART_SIZE);
return conf;
}
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testPartitionPropertyPropagatesToConf() throws Throwable {
assertEquals(1,
getConf().getInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE,
0));
}
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testPartionPropertyPropagatesToStore() throws Throwable {
assertEquals(1, fs.getStore().getPartsizeKB());
}
/**
* tests functionality for big files ( > 5Gb) upload
*/
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testFilePartUpload() throws Throwable {
final Path path = new Path("/test/testFilePartUpload");
int len = 8192;
final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
FSDataOutputStream out = fs.create(path,
false,
getBufferSize(),
(short) 1,
BLOCK_SIZE);
try {
int totalPartitionsToWrite = len / PART_SIZE_BYTES;
assertPartitionsWritten("Startup", out, 0);
//write 2048
int firstWriteLen = 2048;
out.write(src, 0, firstWriteLen);
//assert
long expected = getExpectedPartitionsWritten(firstWriteLen,
PART_SIZE_BYTES,
false);
SwiftUtils.debug(LOG, "First write: predict %d partitions written",
expected);
assertPartitionsWritten("First write completed", out, expected);
//write the rest
int remainder = len - firstWriteLen;
SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
out.write(src, firstWriteLen, remainder);
expected =
getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
assertPartitionsWritten("Remaining data", out, expected);
out.close();
expected =
getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
assertPartitionsWritten("Stream closed", out, expected);
Header[] headers = fs.getStore().getObjectHeaders(path, true);
for (Header header : headers) {
LOG.info(header.toString());
}
byte[] dest = readDataset(fs, path, len);
LOG.info("Read dataset from " + path + ": data length =" + len);
//compare data
SwiftTestUtils.compareByteArrays(src, dest, len);
FileStatus status;
final Path qualifiedPath = path.makeQualified(fs);
status = fs.getFileStatus(qualifiedPath);
//now see what block location info comes back.
//This will vary depending on the Swift version, so the results
//aren't checked -merely that the test actually worked
BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
assertNotNull("Null getFileBlockLocations()", locations);
assertTrue("empty array returned for getFileBlockLocations()",
locations.length > 0);
//last bit of test -which seems to play up on partitions, which we download
//to a skip
try {
validatePathLen(path, len);
} catch (AssertionError e) {
//downgrade to a skip
throw new AssumptionViolatedException(e, null);
}
} finally {
IOUtils.closeStream(out);
}
}
/**
* tests functionality for big files ( > 5Gb) upload
*/
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
final Path path = new Path("/test/testFilePartUploadLengthCheck");
int len = 8192;
final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
FSDataOutputStream out = fs.create(path,
false,
getBufferSize(),
(short) 1,
BLOCK_SIZE);
try {
int totalPartitionsToWrite = len / PART_SIZE_BYTES;
assertPartitionsWritten("Startup", out, 0);
//write 2048
int firstWriteLen = 2048;
out.write(src, 0, firstWriteLen);
//assert
long expected = getExpectedPartitionsWritten(firstWriteLen,
PART_SIZE_BYTES,
false);
SwiftUtils.debug(LOG, "First write: predict %d partitions written",
expected);
assertPartitionsWritten("First write completed", out, expected);
//write the rest
int remainder = len - firstWriteLen;
SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
out.write(src, firstWriteLen, remainder);
expected =
getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
assertPartitionsWritten("Remaining data", out, expected);
out.close();
expected =
getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
assertPartitionsWritten("Stream closed", out, expected);
Header[] headers = fs.getStore().getObjectHeaders(path, true);
for (Header header : headers) {
LOG.info(header.toString());
}
byte[] dest = readDataset(fs, path, len);
LOG.info("Read dataset from " + path + ": data length =" + len);
//compare data
SwiftTestUtils.compareByteArrays(src, dest, len);
FileStatus status = fs.getFileStatus(path);
//now see what block location info comes back.
//This will vary depending on the Swift version, so the results
//aren't checked -merely that the test actually worked
BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
assertNotNull("Null getFileBlockLocations()", locations);
assertTrue("empty array returned for getFileBlockLocations()",
locations.length > 0);
} finally {
IOUtils.closeStream(out);
}
}
private FileStatus validatePathLen(Path path, int len) throws IOException {
//verify that the length is what was written in a direct status check
final Path qualifiedPath = path.makeQualified(fs);
FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
StringBuilder listing = lsToString(parentDirListing);
String parentDirLS = listing.toString();
FileStatus status = fs.getFileStatus(qualifiedPath);
assertEquals("Length of written file " + qualifiedPath
+ " from status check " + status
+ " in dir " + listing,
len,
status.getLen());
String fileInfo = qualifiedPath + " " + status;
assertFalse("File claims to be a directory " + fileInfo,
status.isDir());
FileStatus listedFileStat = resolveChild(parentDirListing, qualifiedPath);
assertNotNull("Did not find " + path + " in " + parentDirLS,
listedFileStat);
//file is in the parent dir. Now validate it's stats
assertEquals("Wrong len for " + path + " in listing " + parentDirLS,
len,
listedFileStat.getLen());
listedFileStat.toString();
return status;
}
private FileStatus resolveChild(FileStatus[] parentDirListing,
Path childPath) {
FileStatus listedFileStat = null;
for (FileStatus stat : parentDirListing) {
if (stat.getPath().equals(childPath)) {
listedFileStat = stat;
}
}
return listedFileStat;
}
private StringBuilder lsToString(FileStatus[] parentDirListing) {
StringBuilder listing = new StringBuilder();
for (FileStatus stat : parentDirListing) {
listing.append(stat).append("\n");
}
return listing;
}
/**
* Calculate the #of partitions expected from the upload
* @param uploaded number of bytes uploaded
* @param partSizeBytes the partition size
* @param closed whether or not the stream has closed
* @return the expected number of partitions, for use in assertions.
*/
private int getExpectedPartitionsWritten(long uploaded,
int partSizeBytes,
boolean closed) {
//#of partitions in total
int partitions = (int) (uploaded / partSizeBytes);
//#of bytes past the last partition
int remainder = (int) (uploaded % partSizeBytes);
if (closed) {
//all data is written, so if there was any remainder, it went up
//too
return partitions + ((remainder > 0) ? 1 : 0);
} else {
//not closed. All the remainder is buffered,
return partitions;
}
}
private int getBufferSize() {
return fs.getConf().getInt("io.file.buffer.size", 4096);
}
/**
* Test sticks up a very large partitioned file and verifies that
* it comes back unchanged.
* @throws Throwable
*/
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
final Path path = new Path("/test/testManyPartitionedFile");
int len = PART_SIZE_BYTES * 15;
final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
FSDataOutputStream out = fs.create(path,
false,
getBufferSize(),
(short) 1,
BLOCK_SIZE);
out.write(src, 0, src.length);
int expected =
getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
out.close();
assertPartitionsWritten("write completed", out, expected);
assertEquals("too few bytes written", len,
SwiftNativeFileSystem.getBytesWritten(out));
assertEquals("too few bytes uploaded", len,
SwiftNativeFileSystem.getBytesUploaded(out));
//now we verify that the data comes back. If it
//doesn't, it means that the ordering of the partitions
//isn't right
byte[] dest = readDataset(fs, path, len);
//compare data
SwiftTestUtils.compareByteArrays(src, dest, len);
//finally, check the data
FileStatus[] stats = fs.listStatus(path);
assertEquals("wrong entry count in "
+ SwiftTestUtils.dumpStats(path.toString(), stats),
expected, stats.length);
}
/**
* Test that when a partitioned file is overwritten by a smaller one,
* all the old partitioned files go away
* @throws Throwable
*/
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testOverwritePartitionedFile() throws Throwable {
final Path path = new Path("/test/testOverwritePartitionedFile");
final int len1 = 8192;
final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
FSDataOutputStream out = fs.create(path,
false,
getBufferSize(),
(short) 1,
1024);
out.write(src1, 0, len1);
out.close();
long expected = getExpectedPartitionsWritten(len1,
PART_SIZE_BYTES,
false);
assertPartitionsWritten("initial upload", out, expected);
assertExists("Exists", path);
FileStatus status = fs.getFileStatus(path);
assertEquals("Length", len1, status.getLen());
//now write a shorter file with a different dataset
final int len2 = 4095;
final byte[] src2 = SwiftTestUtils.dataset(len2, 'a', 'z');
out = fs.create(path,
true,
getBufferSize(),
(short) 1,
1024);
out.write(src2, 0, len2);
out.close();
status = fs.getFileStatus(path);
assertEquals("Length", len2, status.getLen());
byte[] dest = readDataset(fs, path, len2);
//compare data
SwiftTestUtils.compareByteArrays(src2, dest, len2);
}
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeleteSmallPartitionedFile() throws Throwable {
final Path path = new Path("/test/testDeleteSmallPartitionedFile");
final int len1 = 1024;
final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
assertExists("Exists", path);
Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
String ls = SwiftTestUtils.ls(fs, path);
assertExists("Partition 0001 Exists in " + ls, part_0001);
assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
assertExists("Partition 0002 Exists in " + ls, part_0001);
fs.delete(path, false);
assertPathDoesNotExist("deleted file still there", path);
ls = SwiftTestUtils.ls(fs, path);
assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
}
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
final Path path = new Path("/test/testDeletePartitionedFile");
SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
assertExists("Exists", path);
Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
String ls = SwiftTestUtils.ls(fs, path);
assertExists("Partition 0001 Exists in " + ls, part_0001);
assertExists("Partition 0002 Exists in " + ls, part_0001);
fs.delete(path, false);
assertPathDoesNotExist("deleted file still there", path);
ls = SwiftTestUtils.ls(fs, path);
assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testRenamePartitionedFile() throws Throwable {
Path src = new Path("/test/testRenamePartitionedFileSrc");
int len = data.length;
SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
assertExists("Exists", src);
String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
Path srcPart = new Path(src, partOneName);
Path dest = new Path("/test/testRenamePartitionedFileDest");
Path destPart = new Path(src, partOneName);
assertExists("Partition Exists", srcPart);
fs.rename(src, dest);
assertPathExists(fs, "dest file missing", dest);
FileStatus status = fs.getFileStatus(dest);
assertEquals("Length of renamed file is wrong", len, status.getLen());
byte[] destData = readDataset(fs, dest, len);
//compare data
SwiftTestUtils.compareByteArrays(data, destData, len);
String srcLs = SwiftTestUtils.ls(fs, src);
String destLs = SwiftTestUtils.ls(fs, dest);
assertPathDoesNotExist("deleted file still found in " + srcLs, src);
assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
}
}

94
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
import java.io.EOFException;
import java.io.IOException;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
/**
* Test filesystem read operations
*/
public class TestSwiftFileSystemRead extends SwiftFileSystemBaseTest {
/**
* Read past the end of a file: expect the operation to fail
* @throws IOException
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testOverRead() throws IOException {
final String message = "message";
final Path filePath = new Path("/test/file.txt");
writeTextFile(fs, filePath, message, false);
try {
readBytesToString(fs, filePath, 20);
fail("expected an exception");
} catch (EOFException e) {
//expected
}
}
/**
* Read and write some JSON
* @throws IOException
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRWJson() throws IOException {
final String message = "{" +
" 'json': { 'i':43, 'b':true}," +
" 's':'string'" +
"}";
final Path filePath = new Path("/test/file.json");
writeTextFile(fs, filePath, message, false);
String readJson = readBytesToString(fs, filePath, message.length());
assertEquals(message,readJson);
//now find out where it is
FileStatus status = fs.getFileStatus(filePath);
BlockLocation[] locations = fs.getFileBlockLocations(status, 0, 10);
}
/**
* Read and write some XML
* @throws IOException
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRWXML() throws IOException {
final String message = "<x>" +
" <json i='43' 'b'=true/>" +
" string" +
"</x>";
final Path filePath = new Path("/test/file.xml");
writeTextFile(fs, filePath, message, false);
String read = readBytesToString(fs, filePath, message.length());
assertEquals(message,read);
}
}

270
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java Normal file
View File

@ -0,0 +1,270 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.IOException;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.compareByteArrays;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dataset;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeDataset;
public class TestSwiftFileSystemRename extends SwiftFileSystemBaseTest {
/**
* Rename a file into a directory
*
* @throws Exception
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameFileIntoExistingDirectory() throws Exception {
assumeRenameSupported();
Path src = path("/test/olddir/file");
createFile(src);
Path dst = path("/test/new/newdir");
fs.mkdirs(dst);
rename(src, dst, true, false, true);
Path newFile = path("/test/new/newdir/file");
if (!fs.exists(newFile)) {
String ls = ls(dst);
LOG.info(ls(path("/test/new")));
LOG.info(ls(path("/test/hadoop")));
fail("did not find " + newFile + " - directory: " + ls);
}
assertTrue("Destination changed",
fs.exists(path("/test/new/newdir/file")));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameFile() throws Exception {
assumeRenameSupported();
final Path old = new Path("/test/alice/file");
final Path newPath = new Path("/test/bob/file");
fs.mkdirs(newPath.getParent());
final FSDataOutputStream fsDataOutputStream = fs.create(old);
final byte[] message = "Some data".getBytes();
fsDataOutputStream.write(message);
fsDataOutputStream.close();
assertTrue(fs.exists(old));
rename(old, newPath, true, false, true);
final FSDataInputStream bobStream = fs.open(newPath);
final byte[] bytes = new byte[512];
final int read = bobStream.read(bytes);
bobStream.close();
final byte[] buffer = new byte[read];
System.arraycopy(bytes, 0, buffer, 0, read);
assertEquals(new String(message), new String(buffer));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameDirectory() throws Exception {
assumeRenameSupported();
final Path old = new Path("/test/data/logs");
final Path newPath = new Path("/test/var/logs");
fs.mkdirs(old);
fs.mkdirs(newPath.getParent());
assertTrue(fs.exists(old));
rename(old, newPath, true, false, true);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameTheSameDirectory() throws Exception {
assumeRenameSupported();
final Path old = new Path("/test/usr/data");
fs.mkdirs(old);
rename(old, old, false, true, true);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameDirectoryIntoExistingDirectory() throws Exception {
assumeRenameSupported();
Path src = path("/test/olddir/dir");
fs.mkdirs(src);
createFile(path("/test/olddir/dir/file1"));
createFile(path("/test/olddir/dir/subdir/file2"));
Path dst = path("/test/new/newdir");
fs.mkdirs(dst);
//this renames into a child
rename(src, dst, true, false, true);
assertExists("new dir", path("/test/new/newdir/dir"));
assertExists("Renamed nested file1", path("/test/new/newdir/dir/file1"));
assertPathDoesNotExist("Nested file1 should have been deleted",
path("/test/olddir/dir/file1"));
assertExists("Renamed nested subdir",
path("/test/new/newdir/dir/subdir/"));
assertExists("file under subdir",
path("/test/new/newdir/dir/subdir/file2"));
assertPathDoesNotExist("Nested /test/hadoop/dir/subdir/file2 still exists",
path("/test/olddir/dir/subdir/file2"));
}
/**
* trying to rename a directory onto itself should fail,
* preserving everything underneath.
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameDirToSelf() throws Throwable {
assumeRenameSupported();
Path parentdir = path("/test/parentdir");
fs.mkdirs(parentdir);
Path child = new Path(parentdir, "child");
createFile(child);
rename(parentdir, parentdir, false, true, true);
//verify the child is still there
assertIsFile(child);
}
/**
* Assert that root directory renames are not allowed
*
* @throws Exception on failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameRootDirForbidden() throws Exception {
assumeRenameSupported();
rename(path("/"),
path("/test/newRootDir"),
false, true, false);
}
/**
* Assert that renaming a parent directory to be a child
* of itself is forbidden
*
* @throws Exception on failures
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameChildDirForbidden() throws Exception {
assumeRenameSupported();
Path parentdir = path("/test/parentdir");
fs.mkdirs(parentdir);
Path childFile = new Path(parentdir, "childfile");
createFile(childFile);
//verify one level down
Path childdir = new Path(parentdir, "childdir");
rename(parentdir, childdir, false, true, false);
//now another level
fs.mkdirs(childdir);
Path childchilddir = new Path(childdir, "childdir");
rename(parentdir, childchilddir, false, true, false);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameFileAndVerifyContents() throws IOException {
assumeRenameSupported();
final Path filePath = new Path("/test/home/user/documents/file.txt");
final Path newFilePath = new Path("/test/home/user/files/file.txt");
mkdirs(newFilePath.getParent());
int len = 1024;
byte[] dataset = dataset(len, 'A', 26);
writeDataset(fs, filePath, dataset, len, len, false);
rename(filePath, newFilePath, true, false, true);
byte[] dest = readDataset(fs, newFilePath, len);
compareByteArrays(dataset, dest, len);
String reread = readBytesToString(fs, newFilePath, 20);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testMoveFileUnderParent() throws Throwable {
if (!renameSupported()) return;
Path filepath = path("test/file");
createFile(filepath);
//HDFS expects rename src, src -> true
rename(filepath, filepath, true, true, true);
//verify the file is still there
assertIsFile(filepath);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testMoveDirUnderParent() throws Throwable {
if (!renameSupported()) {
return;
}
Path testdir = path("test/dir");
fs.mkdirs(testdir);
Path parent = testdir.getParent();
//the outcome here is ambiguous, so is not checked
fs.rename(testdir, parent);
assertExists("Source directory has been deleted ", testdir);
}
/**
* trying to rename a file onto itself should succeed (it's a no-op)
*/
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameFileToSelf() throws Throwable {
if (!renameSupported()) return;
Path filepath = path("test/file");
createFile(filepath);
//HDFS expects rename src, src -> true
rename(filepath, filepath, true, true, true);
//verify the file is still there
assertIsFile(filepath);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenamedConsistence() throws IOException {
assumeRenameSupported();
describe("verify that overwriting a file with new data doesn't impact" +
" the existing content");
final Path filePath = new Path("/test/home/user/documents/file.txt");
final Path newFilePath = new Path("/test/home/user/files/file.txt");
mkdirs(newFilePath.getParent());
int len = 1024;
byte[] dataset = dataset(len, 'A', 26);
byte[] dataset2 = dataset(len, 'a', 26);
writeDataset(fs, filePath, dataset, len, len, false);
rename(filePath, newFilePath, true, false, true);
SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
byte[] dest = readDataset(fs, newFilePath, len);
compareByteArrays(dataset, dest, len);
String reread = readBytesToString(fs, newFilePath, 20);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameMissingFile() throws Throwable {
assumeRenameSupported();
Path path = path("/test/RenameMissingFile");
Path path2 = path("/test/RenameMissingFileDest");
mkdirs(path("test"));
rename(path, path2, false, false, false);
}
}

158
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java Normal file
View File

@ -0,0 +1,158 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.http.RestClientBindings;
import org.apache.hadoop.fs.swift.http.SwiftRestClient;
import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
import org.apache.hadoop.fs.swift.util.SwiftUtils;
import org.junit.Test;
import java.net.URI;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* Unit tests for SwiftObjectPath class.
*/
public class TestSwiftObjectPath implements SwiftTestConstants {
private static final Log LOG = LogFactory.getLog(TestSwiftObjectPath.class);
/**
* What an endpoint looks like. This is derived from a (valid)
* rackspace endpoint address
*/
private static final String ENDPOINT =
"https://storage101.region1.example.org/v1/MossoCloudFS_9fb40cc0-1234-5678-9abc-def000c9a66";
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testParsePath() throws Exception {
final String pathString = "/home/user/files/file1";
final Path path = new Path(pathString);
final URI uri = new URI("http://container.localhost");
final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
final SwiftObjectPath actual = new SwiftObjectPath(
RestClientBindings.extractContainerName(uri),
pathString);
assertEquals(expected, actual);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testParseUrlPath() throws Exception {
final String pathString = "swift://container.service1/home/user/files/file1";
final URI uri = new URI(pathString);
final Path path = new Path(pathString);
final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
final SwiftObjectPath actual = new SwiftObjectPath(
RestClientBindings.extractContainerName(uri),
"/home/user/files/file1");
assertEquals(expected, actual);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testParseAuthenticatedUrl() throws Exception {
final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx";
final URI uri = new URI(pathString);
final Path path = new Path(pathString);
final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
final SwiftObjectPath actual = new SwiftObjectPath(
RestClientBindings.extractContainerName(uri),
"/home/tom/documents/finance.docx");
assertEquals(expected, actual);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
String initialpath = "/dir/file1";
Path ipath = new Path(initialpath);
SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
ipath);
URI endpoint = new URI(ENDPOINT);
URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
LOG.info("Inital Hadoop Path =" + initialpath);
LOG.info("Merged URI=" + uri);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRootDirProbeEmptyPath() throws Throwable {
SwiftObjectPath object=new SwiftObjectPath("container","");
assertTrue(SwiftUtils.isRootDir(object));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRootDirProbeRootPath() throws Throwable {
SwiftObjectPath object=new SwiftObjectPath("container","/");
assertTrue(SwiftUtils.isRootDir(object));
}
private void assertParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
assertTrue(p1.toString() + " is not a parent of " + p2 ,p1.isEqualToOrParentOf(
p2));
}
private void assertNotParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
assertFalse(p1.toString() + " is a parent of " + p2, p1.isEqualToOrParentOf(
p2));
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testChildOfProbe() throws Throwable {
SwiftObjectPath parent = new SwiftObjectPath("container",
"/parent");
SwiftObjectPath parent2 = new SwiftObjectPath("container",
"/parent2");
SwiftObjectPath child = new SwiftObjectPath("container",
"/parent/child");
SwiftObjectPath sibling = new SwiftObjectPath("container",
"/parent/sibling");
SwiftObjectPath grandchild = new SwiftObjectPath("container",
"/parent/child/grandchild");
assertParentOf(parent, child);
assertParentOf(parent, grandchild);
assertParentOf(child, grandchild);
assertParentOf(parent, parent);
assertNotParentOf(child, parent);
assertParentOf(child, child);
assertNotParentOf(parent, parent2);
assertNotParentOf(grandchild, parent);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testChildOfRoot() throws Throwable {
SwiftObjectPath root = new SwiftObjectPath("container", "/");
SwiftObjectPath child = new SwiftObjectPath("container", "child");
SwiftObjectPath grandchild = new SwiftObjectPath("container",
"/child/grandchild");
assertParentOf(root, child);
assertParentOf(root, grandchild);
assertParentOf(child, grandchild);
assertParentOf(root, root);
assertNotParentOf(child, root);
assertParentOf(child, child);
assertNotParentOf(grandchild, root);
}
}

43
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.hdfs2;
import org.apache.hadoop.fs.swift.TestSwiftFileSystemDirectories;
import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
/**
* Add some HDFS-2 only assertions to {@link TestSwiftFileSystemDirectories}
*/
public class TestSwiftFileSystemDirectoriesHdfs2 extends
TestSwiftFileSystemDirectories {
/**
* make assertions about fields that only appear in
* FileStatus in HDFS2
* @param stat status to look at
*/
protected void extraStatusAssertions(SwiftFileStatus stat) {
//HDFS2
assertTrue("isDirectory(): Not a directory: " + stat, stat.isDirectory());
assertFalse("isFile(): declares itself a file: " + stat, stat.isFile());
assertFalse("isFile(): declares itself a file: " + stat, stat.isSymlink());
}
}

129
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.hdfs2;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
import java.io.IOException;
public class TestV2LsOperations extends SwiftFileSystemBaseTest {
private Path[] testDirs;
/**
* Setup creates dirs under test/hadoop
* @throws Exception
*/
@Override
public void setUp() throws Exception {
super.setUp();
//delete the test directory
Path test = path("/test");
fs.delete(test, true);
mkdirs(test);
}
/**
* Create subdirectories and files under test/ for those tests
* that want them. Doing so adds overhead to setup and teardown,
* so should only be done for those tests that need them.
* @throws IOException on an IO problem
*/
private void createTestSubdirs() throws IOException {
testDirs = new Path[]{
path("/test/hadoop/a"),
path("/test/hadoop/b"),
path("/test/hadoop/c/1"),
};
assertPathDoesNotExist("test directory setup", testDirs[0]);
for (Path path : testDirs) {
mkdirs(path);
}
}
/**
* To get this project to compile under Hadoop 1, this code needs to be
* commented out
*
*
* @param fs filesystem
* @param dir dir
* @param subdir subdir
* @param recursive recurse?
* @throws IOException IO problems
*/
public static void assertListFilesFinds(FileSystem fs,
Path dir,
Path subdir,
boolean recursive) throws IOException {
RemoteIterator<LocatedFileStatus> iterator =
fs.listFiles(dir, recursive);
boolean found = false;
int entries = 0;
StringBuilder builder = new StringBuilder();
while (iterator.hasNext()) {
LocatedFileStatus next = iterator.next();
entries++;
builder.append(next.toString()).append('\n');
if (next.getPath().equals(subdir)) {
found = true;
}
}
assertTrue("Path " + subdir
+ " not found in directory " + dir + " : "
+ " entries=" + entries
+ " content"
+ builder.toString(),
found);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListFilesRootDir() throws Throwable {
Path dir = path("/");
Path child = new Path(dir, "test");
fs.delete(child, true);
SwiftTestUtils.writeTextFile(fs, child, "text", false);
assertListFilesFinds(fs, dir, child, false);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListFilesSubDir() throws Throwable {
createTestSubdirs();
Path dir = path("/test");
Path child = new Path(dir, "text.txt");
SwiftTestUtils.writeTextFile(fs, child, "text", false);
assertListFilesFinds(fs, dir, child, false);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testListFilesRecursive() throws Throwable {
createTestSubdirs();
Path dir = path("/test");
Path child = new Path(dir, "hadoop/a/a.txt");
SwiftTestUtils.writeTextFile(fs, child, "text", false);
assertListFilesFinds(fs, dir, child, true);
}
}

198
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java Normal file
View File

@ -0,0 +1,198 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.swift.SwiftTestConstants;
import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Properties;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_AUTH_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONTAINER_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTPS_PORT_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTP_PORT_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PASSWORD_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_REGION_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_TENANT_PROPERTY;
import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_USERNAME_PROPERTY;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPropertyEquals;
public class TestRestClientBindings extends Assert
implements SwiftTestConstants {
private static final String SERVICE = "sname";
private static final String CONTAINER = "cname";
private static final String FS_URI = "swift://"
+ CONTAINER + "." + SERVICE + "/";
private static final String AUTH_URL = "http://localhost:8080/auth";
private static final String USER = "user";
private static final String PASS = "pass";
private static final String TENANT = "tenant";
private URI filesysURI;
private Configuration conf;
@Before
public void setup() throws URISyntaxException {
filesysURI = new URI(FS_URI);
conf = new Configuration(true);
setInstanceVal(conf, SERVICE, DOT_AUTH_URL, AUTH_URL);
setInstanceVal(conf, SERVICE, DOT_USERNAME, USER);
setInstanceVal(conf, SERVICE, DOT_PASSWORD, PASS);
}
private void setInstanceVal(Configuration conf,
String host,
String key,
String val) {
String instance = RestClientBindings.buildSwiftInstancePrefix(host);
String confkey = instance
+ key;
conf.set(confkey, val);
}
public void testPrefixBuilder() throws Throwable {
String built = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
assertEquals("fs.swift.service." + SERVICE, built);
}
public void testBindAgainstConf() throws Exception {
Properties props = RestClientBindings.bind(filesysURI, conf);
assertPropertyEquals(props, SWIFT_CONTAINER_PROPERTY, CONTAINER);
assertPropertyEquals(props, SWIFT_SERVICE_PROPERTY, SERVICE);
assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
assertPropertyEquals(props, SWIFT_USERNAME_PROPERTY, USER);
assertPropertyEquals(props, SWIFT_PASSWORD_PROPERTY, PASS);
assertPropertyEquals(props, SWIFT_TENANT_PROPERTY, null);
assertPropertyEquals(props, SWIFT_REGION_PROPERTY, null);
assertPropertyEquals(props, SWIFT_HTTP_PORT_PROPERTY, null);
assertPropertyEquals(props, SWIFT_HTTPS_PORT_PROPERTY, null);
}
public void expectBindingFailure(URI fsURI, Configuration config) {
try {
Properties binding = RestClientBindings.bind(fsURI, config);
//if we get here, binding didn't fail- there is something else.
//list the properties but not the values.
StringBuilder details = new StringBuilder() ;
for (Object key: binding.keySet()) {
details.append(key.toString()).append(" ");
}
fail("Expected a failure, got the binding [ "+ details+"]");
} catch (SwiftConfigurationException expected) {
}
}
public void testBindAgainstConfMissingInstance() throws Exception {
Configuration badConf = new Configuration();
expectBindingFailure(filesysURI, badConf);
}
/* Hadoop 2.x+ only, as conf.unset() isn't a v1 feature
public void testBindAgainstConfIncompleteInstance() throws Exception {
String instance = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
conf.unset(instance + DOT_PASSWORD);
expectBindingFailure(filesysURI, conf);
}
*/
@Test(expected = SwiftConfigurationException.class)
public void testDottedServiceURL() throws Exception {
RestClientBindings.bind(new URI("swift://hadoop.apache.org/"), conf);
}
@Test(expected = SwiftConfigurationException.class)
public void testMissingServiceURL() throws Exception {
RestClientBindings.bind(new URI("swift:///"), conf);
}
/**
* inner test method that expects container extraction to fail
* -if not prints a meaningful error message.
*
* @param hostname hostname to parse
*/
private static void expectExtractContainerFail(String hostname) {
try {
String container = RestClientBindings.extractContainerName(hostname);
fail("Expected an error -got a container of '" + container
+ "' from " + hostname);
} catch (SwiftConfigurationException expected) {
//expected
}
}
/**
* inner test method that expects service extraction to fail
* -if not prints a meaningful error message.
*
* @param hostname hostname to parse
*/
public static void expectExtractServiceFail(String hostname) {
try {
String service = RestClientBindings.extractServiceName(hostname);
fail("Expected an error -got a service of '" + service
+ "' from " + hostname);
} catch (SwiftConfigurationException expected) {
//expected
}
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testEmptyHostname() throws Throwable {
expectExtractContainerFail("");
expectExtractServiceFail("");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDot() throws Throwable {
expectExtractContainerFail(".");
expectExtractServiceFail(".");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSimple() throws Throwable {
expectExtractContainerFail("simple");
expectExtractServiceFail("simple");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testTrailingDot() throws Throwable {
expectExtractServiceFail("simple.");
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLeadingDot() throws Throwable {
expectExtractServiceFail(".leading");
}
}

117
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.http;
import org.apache.commons.httpclient.Header;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.SwiftTestConstants;
import org.apache.hadoop.fs.swift.util.Duration;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
public class TestSwiftRestClient implements SwiftTestConstants {
private static final Log LOG =
LogFactory.getLog(TestSwiftRestClient.class);
private Configuration conf;
private boolean runTests;
private URI serviceURI;
@Before
public void setup() throws IOException {
conf = new Configuration();
runTests = SwiftTestUtils.hasServiceURI(conf);
if (runTests) {
serviceURI = SwiftTestUtils.getServiceURI(conf);
}
}
protected void assumeEnabled() {
Assume.assumeTrue(runTests);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCreate() throws Throwable {
assumeEnabled();
SwiftRestClient client = createClient();
}
private SwiftRestClient createClient() throws IOException {
return SwiftRestClient.getInstance(serviceURI, conf);
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testAuthenticate() throws Throwable {
assumeEnabled();
SwiftRestClient client = createClient();
client.authenticate();
}
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testPutAndDelete() throws Throwable {
assumeEnabled();
SwiftRestClient client = createClient();
client.authenticate();
Path path = new Path("restTestPutAndDelete");
SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path);
byte[] stuff = new byte[1];
stuff[0] = 'a';
client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length);
//check file exists
Duration head = new Duration();
Header[] responseHeaders = client.headRequest("expect success",
sobject,
SwiftRestClient.NEWEST);
head.finished();
LOG.info("head request duration " + head);
for (Header header: responseHeaders) {
LOG.info(header.toString());
}
//delete the file
client.delete(sobject);
//check file is gone
try {
Header[] headers = client.headRequest("expect fail",
sobject,
SwiftRestClient.NEWEST);
Assert.fail("Expected deleted file, but object is still present: "
+ sobject);
} catch (FileNotFoundException e) {
//expected
}
for (DurationStats stats: client.getOperationStatistics()) {
LOG.info(stats);
}
}
}

37
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.scale;
import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
/**
* Base class for scale tests; here is where the common scale configuration
* keys are defined
*/
public class SwiftScaleTestBase extends SwiftFileSystemBaseTest {
public static final String SCALE_TEST = "scale.test.";
public static final String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count";
public static final long DEFAULT_OPERATION_COUNT = 10;
protected long getOperationCount() {
return getConf().getLong(KEY_OPERATION_COUNT, DEFAULT_OPERATION_COUNT);
}
}

96
hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java Normal file
View File

@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.swift.scale;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.util.Duration;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.Test;
public class TestWriteManySmallFiles extends SwiftScaleTestBase {
public static final Log LOG = LogFactory.getLog(TestWriteManySmallFiles.class);
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testScaledWriteThenRead() throws Throwable {
Path dir = new Path("/test/manysmallfiles");
Duration rm1 = new Duration();
fs.delete(dir, true);
rm1.finished();
fs.mkdirs(dir);
Duration ls1 = new Duration();
fs.listStatus(dir);
ls1.finished();
long count = getOperationCount();
SwiftTestUtils.noteAction("Beginning Write of "+ count + " files ");
DurationStats writeStats = new DurationStats("write");
DurationStats readStats = new DurationStats("read");
String format = "%08d";
for (long l = 0; l < count; l++) {
String name = String.format(format, l);
Path p = new Path(dir, "part-" + name);
Duration d = new Duration();
SwiftTestUtils.writeTextFile(fs, p, name, false);
d.finished();
writeStats.add(d);
Thread.sleep(1000);
}
//at this point, the directory is full.
SwiftTestUtils.noteAction("Beginning ls");
Duration ls2 = new Duration();
FileStatus[] status2 = (FileStatus[]) fs.listStatus(dir);
ls2.finished();
assertEquals("Not enough entries in the directory", count, status2.length);
SwiftTestUtils.noteAction("Beginning read");
for (long l = 0; l < count; l++) {
String name = String.format(format, l);
Path p = new Path(dir, "part-" + name);
Duration d = new Duration();
String result = SwiftTestUtils.readBytesToString(fs, p, name.length());
assertEquals(name, result);
d.finished();
readStats.add(d);
}
//do a recursive delete
SwiftTestUtils.noteAction("Beginning delete");
Duration rm2 = new Duration();
fs.delete(dir, true);
rm2.finished();
//print the stats
LOG.info(String.format("'filesystem','%s'",fs.getUri()));
LOG.info(writeStats.toString());
LOG.info(readStats.toString());
LOG.info(String.format(
"'rm1',%d,'ls1',%d",
rm1.value(),
ls1.value()));
LOG.info(String.format(
"'rm2',%d,'ls2',%d",
rm2.value(),
ls2.value()));
}
}

51
hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml Normal file
View File

@ -0,0 +1,51 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- Values used when running unit tests. This is mostly empty, to -->
<!-- use of the default values, overriding the potentially -->
<!-- user-editted core-site.xml in the conf/ directory. -->
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>target/build/test</value>
<description>A base for other temporary directories.</description>
<final>true</final>
</property>
<!-- Turn security off for tests by default -->
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
<!--
To run these tests.
# Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
# add the property test.fs.swift.name to point to a swift filesystem URL
# Add the credentials for the service you are testing against
-->
<include xmlns="http://www.w3.org/2001/XInclude"
href="auth-keys.xml"/>
</configuration>

42
hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties Normal file
View File

@ -0,0 +1,42 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# log4j configuration used during build and unit tests
log4j.rootLogger=INFO,stdout
log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c %x - %m%n"
#log4j.logger.org.apache.hadoop.fs.swift=DEBUG
#crank back on warnings about -1 content length GETs
log4j.logger.org.apache.commons.httpclient.HttpMethodBase=ERROR

6
hadoop-tools/hadoop-tools-dist/pom.xml
View File

@ -77,6 +77,12 @@
<type>pom</type>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-openstack</artifactId>
<scope>compile</scope>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

1
hadoop-tools/pom.xml
View File

@ -40,6 +40,7 @@
<module>hadoop-tools-dist</module>
<module>hadoop-extras</module>
<module>hadoop-pipes</module>
<module>hadoop-openstack</module>
</modules>
<build>