big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7250. Update Shared cache client api to use URLs.

Browse Source
This commit is contained in:
Chris Trezzo 2017-09-28 15:28:06 -07:00
parent 6f789fe057
commit c114da5e64
3 changed files with 23 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.impl.SharedCacheClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -58,34 +59,25 @@ public SharedCacheClient(String name) {
*
* <p>
* The <code>SharedCacheManager</code> responds with whether or not the
* resource exists in the cache. If the resource exists, a <code>Path</code>
* to the resource in the shared cache is returned. If the resource does not
* resource exists in the cache. If the resource exists, a <code>URL</code> to
* the resource in the shared cache is returned. If the resource does not
* exist, null is returned instead.
* </p>
*
* <p>
* Once a path has been returned for a resource, that path is safe to use for
* Once a URL has been returned for a resource, that URL is safe to use for
* the lifetime of the application that corresponds to the provided
* ApplicationId.
* </p>
*
* <p>
* Additionally, a name for the resource should be specified. A fragment will
* be added to the path with the desired name if the desired name is different
* than the name of the provided path from the shared cache. This ensures that
* if the returned path is used to create a LocalResource, then the symlink
* created during YARN localization will match the name specified.
* </p>
*
* @param applicationId ApplicationId of the application using the resource
* @param resourceKey the key (i.e. checksum) that identifies the resource
* @param resourceName the desired name of the resource
* @return Path to the resource, or null if it does not exist
* @return URL to the resource, or null if it does not exist
*/
@Public
@Unstable
public abstract Path use(ApplicationId applicationId, String resourceKey,
String resourceName) throws YarnException;
public abstract URL use(ApplicationId applicationId, String resourceKey)
throws YarnException;
/**
* <p>

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
View File

@ -21,8 +21,6 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -38,6 +36,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.SharedCacheClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -113,8 +112,8 @@ protected void stopClientProxy() {
}
@Override
public Path use(ApplicationId applicationId, String resourceKey,
String resourceName) throws YarnException {
public URL use(ApplicationId applicationId, String resourceKey)
throws YarnException {
Path resourcePath = null;
UseSharedCacheResourceRequest request = Records.newRecord(
UseSharedCacheResourceRequest.class);
@ -132,32 +131,13 @@ public Path use(ApplicationId applicationId, String resourceKey,
throw new YarnException(e);
}
if (resourcePath != null) {
if (resourcePath.getName().equals(resourceName)) {
// The preferred name is the same as the name of the item in the cache,
// so we skip generating the fragment to save space in the MRconfig.
return resourcePath;
URL pathURL = URL.fromPath(resourcePath);
return pathURL;
} else {
// We are using the shared cache, and a preferred name has been
// specified that is different than the name of the resource in the
// shared cache. We need to set the fragment portion of the URI to
// preserve the desired name.
URI pathURI = resourcePath.toUri();
try {
// We assume that there is no existing fragment in the URI since the
// shared cache manager does not use fragments.
pathURI =
new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(),
resourceName);
resourcePath = new Path(pathURI);
} catch (URISyntaxException e) {
throw new YarnException(
"Could not create a new URI due to syntax errors: "
+ pathURI.toString(), e);
// The resource was not in the cache.
return null;
}
}
}
return resourcePath;
}
@Override
public void release(ApplicationId applicationId, String resourceKey)

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
View File

@ -27,7 +27,6 @@
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -40,6 +39,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
import org.junit.AfterClass;
@ -114,36 +114,21 @@ public void testUseCacheMiss() throws Exception {
response.setPath(null);
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
Path newPath = client.use(mock(ApplicationId.class), "key", null);
assertNull("The path is not null!", newPath);
URL newURL = client.use(mock(ApplicationId.class), "key");
assertNull("The path is not null!", newURL);
}
@Test
public void testUseWithResourceName() throws Exception {
public void testUseCacheHit() throws Exception {
Path file = new Path("viewfs://test/path");
URI useUri = new URI("viewfs://test/path#linkName");
Path usePath = new Path(useUri);
URL useUrl = URL.fromPath(new Path("viewfs://test/path"));
UseSharedCacheResourceResponse response =
new UseSharedCacheResourceResponsePBImpl();
response.setPath(file.toString());
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
Path newPath = client.use(mock(ApplicationId.class), "key", "linkName");
assertEquals("The paths are not equal!", usePath, newPath);
}
@Test
public void testUseWithSameResourceName() throws Exception {
Path file = new Path("viewfs://test/path");
URI useUri = new URI("viewfs://test/path");
Path usePath = new Path(useUri);
UseSharedCacheResourceResponse response =
new UseSharedCacheResourceResponsePBImpl();
response.setPath(file.toString());
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
Path newPath = client.use(mock(ApplicationId.class), "key", "path");
assertEquals("The paths are not equal!", usePath, newPath);
URL newURL = client.use(mock(ApplicationId.class), "key");
assertEquals("The paths are not equal!", useUrl, newURL);
}
@Test(expected = YarnException.class)
@ -151,7 +136,7 @@ public void testUseError() throws Exception {
String message = "Mock IOExcepiton!";
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
new IOException(message));
client.use(mock(ApplicationId.class), "key", null);
client.use(mock(ApplicationId.class), "key");
}
@Test