big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9965. Fix NodeManager failing to start on subsequent times when Hdfs Auxillary Jar is set (addendum). Contributed by Prabhu Joseph.

Browse Source
This commit is contained in:
Abhishek Modi 2019-11-19 08:08:40 +05:30
parent 4050471b45
commit dc3f4fc2f4
2 changed files with 43 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
View File

@ -311,7 +311,8 @@ private AuxiliaryService createAuxService(AuxServiceRecord service,
* @return path of the downloaded file
* @throws IOException
*/
private Path maybeDownloadJars(String sName, String className, String
@VisibleForTesting
protected Path maybeDownloadJars(String sName, String className, String
remoteFile, AuxServiceFile.TypeEnum type, Configuration conf)
throws IOException {
// load AuxiliaryService from remote classpath

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
View File

@ -22,6 +22,7 @@
import static org.apache.hadoop.service.Service.STATE.STARTED;
import static org.apache.hadoop.service.Service.STATE.STOPPED;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -95,6 +96,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceFile;
import org.junit.Assert;
import org.junit.Test;
@ -487,6 +489,45 @@ public void testCustomizedAuxServiceClassPath() throws Exception {
}
}
@Test (timeout = 15000)
public void testReuseLocalizedAuxiliaryJar() throws Exception {
File testJar = null;
AuxServices aux = null;
Configuration conf = new YarnConfiguration();
FileSystem fs = FileSystem.get(conf);
String root = "target/LocalDir";
try {
testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), rootDir,
"test-runjar.jar", 2048, ServiceB.class.getName(), LightService
.class.getName());
Context mockContext = mock(Context.class);
LocalDirsHandlerService mockDirsHandler = mock(
LocalDirsHandlerService.class);
Path rootAuxServiceDirPath = new Path(root, "nmAuxService");
when(mockDirsHandler.getLocalPathForWrite(anyString())).thenReturn(
rootAuxServiceDirPath);
when(mockContext.getLocalDirsHandler()).thenReturn(mockDirsHandler);
aux = new AuxServices(MOCK_AUX_PATH_HANDLER, mockContext,
MOCK_DEL_SERVICE);
// First Time the jar gets localized
Path path = aux.maybeDownloadJars("ServiceB", ServiceB.class.getName(),
testJar.getAbsolutePath(), AuxServiceFile.TypeEnum.STATIC, conf);
// Validate the path on reuse of localized jar
path = aux.maybeDownloadJars("ServiceB", ServiceB.class.getName(),
testJar.getAbsolutePath(), AuxServiceFile.TypeEnum.STATIC, conf);
assertFalse("Failed to reuse the localized jar",
path.toString().endsWith("/*"));
} finally {
if (testJar != null) {
testJar.delete();
}
if (fs.exists(new Path(root))) {
fs.delete(new Path(root), true);
}
}
}
@Test
public void testAuxEventDispatch() throws IOException {
Configuration conf = new Configuration();