big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9557. Application fails in diskchecker when ReadWriteDiskValidator is configured. Contributed by Bilwa S T.

Browse Source
This commit is contained in:
bibinchundatt 2019-06-11 23:20:28 +05:30
parent 60c95e9b6a
commit 2263ead365
2 changed files with 38 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -133,7 +133,7 @@ public ContainerLocalizer(FileContext lfs, String user, String appId,
this.localDirs = localDirs;
this.localizerId = localizerId;
this.recordFactory = recordFactory;
this.conf = new YarnConfiguration();
this.conf = initConfiguration();
this.diskValidator = DiskValidatorFactory.getInstance(
YarnConfiguration.DEFAULT_DISK_VALIDATOR);
this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
@ -142,6 +142,12 @@ public ContainerLocalizer(FileContext lfs, String user, String appId,
"token file name cannot be null");
}
@VisibleForTesting
@Private
Configuration initConfiguration() {
return new YarnConfiguration();
}
@Private
@VisibleForTesting
public LocalizationProtocol getProxy(final InetSocketAddress nmAddr) {
@ -250,7 +256,8 @@ Callable<Path> download(Path destDirPath, LocalResource rsrc,
if (rsrc.getVisibility() == LocalResourceVisibility.PRIVATE) {
createParentDirs(destDirPath);
}
diskValidator.checkStatus(new File(destDirPath.toUri().getRawPath()));
diskValidator
.checkStatus(new File(destDirPath.getParent().toUri().getRawPath()));
return new FSDownloadWrapper(lfs, ugi, conf, destDirPath, rsrc);
}

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
View File

@ -70,12 +70,14 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@ -235,6 +237,33 @@ public void testMainFailure() throws Exception {
}
}
@Test
public void testDiskCheckFailure() throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.DISK_VALIDATOR, "read-write");
FileContext lfs = FileContext.getLocalFSFileContext(conf);
Path fileCacheDir = lfs.makeQualified(new Path(basedir, "filecache"));
lfs.mkdir(fileCacheDir, FsPermission.getDefault(), true);
RecordFactory recordFactory = mock(RecordFactory.class);
ContainerLocalizer localizer = new ContainerLocalizer(lfs,
UserGroupInformation.getCurrentUser().getUserName(), "application_01",
"container_01", String.format(ContainerExecutor.TOKEN_FILE_NAME_FMT,
"container_01"), new ArrayList<>(), recordFactory){
@Override
Configuration initConfiguration() {
return conf;
}
};
LocalResource rsrc = mock(LocalResource.class);
Path destDirPath = new Path(fileCacheDir, "11");
try {
localizer.download(destDirPath, rsrc,
UserGroupInformation.getCurrentUser());
} catch (DiskErrorException ex) {
fail(ex.getCause().toString());
}
}
@Test
@SuppressWarnings("unchecked")
public void testLocalizerTokenIsGettingRemoved() throws Exception {