big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-525. Support virtual-hosted style URLs. Contributed by Bharat Viswanadham

Browse Source
This commit is contained in:
Bharat Viswanadham 2018-10-01 13:16:08 -07:00
parent 59d5af21b7
commit 4eff629ab3
7 changed files with 411 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -1035,12 +1035,12 @@
<property>
<name>hadoop.tags.custom</name>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY</value>
</property>
<property>
<name>ozone.tags.system</name>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY</value>
</property>
@ -1222,4 +1222,78 @@
</description>
</property>
<property>
<name>ozone.s3g.authentication.kerberos.principal</name>
<value/>
<tag>OZONE, S3GATEWAY</tag>
<description>The server principal used by Ozone S3Gateway server. This is
typically set to
HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
HTTP/ by convention.</description>
</property>
<property>
<name>ozone.s3g.domain.name</name>
<value/>
<tag>OZONE, S3GATEWAY</tag>
<description>List of Ozone S3Gateway domain names. If multiple
domain names to be provided, they should be a "," seperated.
This parameter is only required when virtual host style pattern is
followed.</description>
</property>
<property>
<name>ozone.s3g.http-address</name>
<value>0.0.0.0:9878</value>
<tag>OZONE, S3GATEWAY</tag>
<description>The address and the base port where the Ozone S3Gateway
Server will
listen on.</description>
</property>
<property>
<name>ozone.s3g.http-bind-host</name>
<value>0.0.0.0</value>
<tag>OZONE, S3GATEWAY</tag>
<description>The actual address the HTTP server will bind to. If this optional address
is set, it overrides only the hostname portion of ozone.s3g.http-address.
This is useful for making the Ozone S3Gateway HTTP server listen on all
interfaces by setting it to 0.0.0.0.</description>
</property>
<property>
<name>ozone.s3g.http.enabled</name>
<value>true</value>
<tag>OZONE, S3GATEWAY</tag>
<description>The boolean which enables the Ozone S3Gateway server
.</description>
</property>
<property>
<name>ozone.s3g.https-address</name>
<value/>
<tag>OZONE, S3GATEWAY</tag>
<description>Ozone S3Gateway serverHTTPS server address and port
.</description>
</property>
<property>
<name>ozone.s3g.https-bind-host</name>
<value/>
<tag>OZONE, S3GATEWAY</tag>
<description>The actual address the HTTPS server will bind to. If this optional address
is set, it overrides only the hostname portion of ozone.s3g.https-address.
This is useful for making the Ozone S3Gateway HTTPS server listen on all
interfaces by setting it to 0.0.0.0.</description>
</property>
<property>
<name>ozone.s3g.keytab.file</name>
<value/>
<tag>OZONE, S3GATEWAY</tag>
<description>The keytab file used by the S3Gateway server to login as its
service principal. </description>
</property>
</configuration>

4
hadoop-ozone/integration-test/pom.xml
View File

@ -42,6 +42,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-objectstore-service</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-s3gateway</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
View File

@ -21,6 +21,7 @@
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
/**
* Tests if configuration constants documented in ozone-defaults.xml.
@ -32,7 +33,8 @@ public void initializeMemberVariables() {
xmlFilename = new String("ozone-default.xml");
configurationClasses =
new Class[] {OzoneConfigKeys.class, ScmConfigKeys.class,
OMConfigKeys.class, HddsConfigKeys.class};
OMConfigKeys.class, HddsConfigKeys.class,
S3GatewayConfigKeys.class};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");

1
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
View File

@ -44,6 +44,7 @@ public final class S3GatewayConfigKeys {
public static final int OZONE_S3G_HTTPS_BIND_PORT_DEFAULT = 9879;
public static final String OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
"ozone.s3g.authentication.kerberos.principal";
public static final String OZONE_S3G_DOMAIN_NAME = "ozone.s3g.domain.name";
/**
* Never constructed.

143
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java Normal file
View File

@ -0,0 +1,143 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.s3;
import javax.inject.Inject;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.container.PreMatching;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.InvalidRequestException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME;
/**
* Filter used to convert virtual host style pattern to path style pattern.
*/
@Provider
@PreMatching
public class VirtualHostStyleFilter implements ContainerRequestFilter {
private static final Logger LOG = LoggerFactory.getLogger(
VirtualHostStyleFilter.class);
private static final Pattern URL_SCHEME_PATTERN = Pattern.compile("" +
"(?<bucket>(.+))\\.(?<volume>(.+))\\.");
@Inject
private OzoneConfiguration conf;
private String[] domains;
@Override
public void filter(ContainerRequestContext requestContext) throws
IOException {
domains = conf.getTrimmedStrings(OZONE_S3G_DOMAIN_NAME);
if (domains.length == 0) {
// domains is not configured, might be it is path style.
// So, do not continue further, just return.
return;
}
//Get the value of the host
String host = requestContext.getHeaderString(HttpHeaders.HOST);
String domain = getDomainName(host);
if (domain == null) {
throw getException("Invalid S3 Gateway request {" + requestContext
.getUriInfo().getRequestUri().toString() + " }: No matching domain " +
"{" + Arrays.toString(domains) + "} for the host {" + host + "}");
}
LOG.debug("Http header host name is {}", host);
LOG.debug("Domain name matched is {}", domain);
//Check if we have a Virtual Host style request, host length greater than
// address length means it is virtual host style, we need to convert to
// path style.
if (host.length() > domain.length()) {
String bothNames = host.substring(0, host.length() - domain.length());
LOG.debug("Both volume name and bucket name is {}", bothNames);
Matcher matcher = URL_SCHEME_PATTERN.matcher(bothNames);
if (!matcher.matches()) {
throw getException("Invalid S3 Gateway request {" + requestContext
.getUriInfo().getRequestUri().toString() +"}:" +" Host: {" + host
+ " is in invalid format");
}
String bucketStr = matcher.group("bucket");
String volumeStr = matcher.group("volume");
LOG.debug("bucket {}, volumeStr {}", bucketStr, volumeStr);
URI baseURI = requestContext.getUriInfo().getBaseUri();
String currentPath = requestContext.getUriInfo().getPath();
String newPath = String.format("%s/%s", volumeStr, bucketStr);
if (currentPath != null) {
newPath += String.format("%s", currentPath);
}
URI requestAddr = UriBuilder.fromUri(baseURI).path(newPath).build();
requestContext.setRequestUri(baseURI, requestAddr);
}
}
private InvalidRequestException getException(String message) {
return new InvalidRequestException(message);
}
@VisibleForTesting
public void setConfiguration(OzoneConfiguration config) {
this.conf = config;
}
/**
* This method finds the longest match with the domain name.
* @param host
* @return domain name matched with the host. if none of them are matching,
* return null.
*/
private String getDomainName(String host) {
String match = null;
int length=0;
for (String domainVal : domains) {
if (host.endsWith(domainVal)) {
int len = domainVal.length();
if (len > length) {
length = len;
match = domainVal;
}
}
}
return match;
}
}

163
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java Normal file
View File

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.s3;
import org.apache.hadoop.fs.InvalidRequestException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.test.GenericTestUtils;
import org.glassfish.jersey.internal.PropertiesDelegate;
import org.glassfish.jersey.server.ContainerRequest;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.SecurityContext;
import java.net.URI;
/**
* This class test virtual host style mapping conversion to path style.
*/
public class TestVirtualHostStyleFilter {
private static OzoneConfiguration conf;
private static String s3HttpAddr;
@Before
public void setup() {
conf = new OzoneConfiguration();
s3HttpAddr = "localhost:9878";
conf.set(S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY, s3HttpAddr);
conf.set(S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME, s3HttpAddr);
}
/**
* Create containerRequest object.
* @return ContainerRequest
* @throws Exception
*/
public ContainerRequest createContainerRequest(String host, String path,
boolean virtualHostStyle)
throws Exception {
URI baseUri = new URI("http://" + s3HttpAddr);
URI virtualHostStyleUri;
if (path == null) {
virtualHostStyleUri = new URI("http://" + s3HttpAddr);
} else {
virtualHostStyleUri = new URI("http://" + s3HttpAddr + path);
}
URI pathStyleUri = new URI("http://" + s3HttpAddr + path);
String httpMethod = "DELETE";
SecurityContext securityContext = Mockito.mock(SecurityContext.class);
PropertiesDelegate propertiesDelegate = Mockito.mock(PropertiesDelegate
.class);
ContainerRequest containerRequest;
if (virtualHostStyle) {
containerRequest = new ContainerRequest(baseUri, virtualHostStyleUri,
httpMethod, securityContext, propertiesDelegate);
containerRequest.header(HttpHeaders.HOST, host);
} else {
containerRequest = new ContainerRequest(baseUri, pathStyleUri,
httpMethod, securityContext, propertiesDelegate);
containerRequest.header(HttpHeaders.HOST, host);
}
return containerRequest;
}
@Test
public void testVirtualHostStyle() throws Exception {
VirtualHostStyleFilter virtualHostStyleFilter =
new VirtualHostStyleFilter();
virtualHostStyleFilter.setConfiguration(conf);
ContainerRequest containerRequest = createContainerRequest("mybucket" +
".myvolume.localhost:9878", "/myfile", true);
virtualHostStyleFilter.filter(containerRequest);
URI expected = new URI("http://" + s3HttpAddr +
"/myvolume/mybucket/myfile");
Assert.assertEquals(expected, containerRequest.getRequestUri());
}
@Test
public void testPathStyle() throws Exception {
VirtualHostStyleFilter virtualHostStyleFilter =
new VirtualHostStyleFilter();
virtualHostStyleFilter.setConfiguration(conf);
ContainerRequest containerRequest = createContainerRequest(s3HttpAddr,
"/myvolume/mybucket/myfile", false);
virtualHostStyleFilter.filter(containerRequest);
URI expected = new URI("http://" + s3HttpAddr +
"/myvolume/mybucket/myfile");
Assert.assertEquals(expected, containerRequest.getRequestUri());
}
@Test
public void testVirtualHostStyleWithCreateBucketRequest() throws Exception {
VirtualHostStyleFilter virtualHostStyleFilter =
new VirtualHostStyleFilter();
virtualHostStyleFilter.setConfiguration(conf);
ContainerRequest containerRequest = createContainerRequest("mybucket" +
".myvolume.localhost:9878", null, true);
virtualHostStyleFilter.filter(containerRequest);
URI expected = new URI("http://" + s3HttpAddr +
"/myvolume/mybucket");
Assert.assertEquals(expected, containerRequest.getRequestUri());
}
@Test
public void testVirtualHostStyleWithNoMatchingDomain() throws Exception {
VirtualHostStyleFilter virtualHostStyleFilter =
new VirtualHostStyleFilter();
virtualHostStyleFilter.setConfiguration(conf);
ContainerRequest containerRequest = createContainerRequest("mybucket" +
".myvolume.localhost:9999", null, true);
try {
virtualHostStyleFilter.filter(containerRequest);
} catch (InvalidRequestException ex) {
GenericTestUtils.assertExceptionContains("No matching domain", ex);
}
}
@Test
public void testVirtualHostStyleWithoutVolumeName() throws Exception {
VirtualHostStyleFilter virtualHostStyleFilter =
new VirtualHostStyleFilter();
virtualHostStyleFilter.setConfiguration(conf);
ContainerRequest containerRequest = createContainerRequest("mybucket." +
".localhost:9878", null, true);
try {
virtualHostStyleFilter.filter(containerRequest);
} catch (InvalidRequestException ex) {
GenericTestUtils.assertExceptionContains("invalid format", ex);
}
}
}

21
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/package-info.java Normal file
View File

@ -0,0 +1,21 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Unit tests for the bucket related rest endpoints.
*/
package org.apache.hadoop.ozone.s3;