big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11463. Node Labels root directory creation doesn't have a retry logic - 2nd addendum (#5670)

Browse Source
This commit is contained in:
Peter Szucs 2023-05-18 14:48:43 +02:00 committed by GitHub
parent 78cc528739
commit ff8eac517a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
View File

@ -65,6 +65,12 @@ protected void initStore(Configuration conf, Path fsStorePath,
this.fsWorkingPath = fsStorePath; this.fsWorkingPath = fsStorePath;
this.manager = mgr; this.manager = mgr;
initFileSystem(conf); initFileSystem(conf);
initNodeStoreRootDirectory(conf);
this.replication = conf.getInt(YarnConfiguration.FS_STORE_FILE_REPLICATION,
YarnConfiguration.DEFAULT_FS_STORE_FILE_REPLICATION);
}
private void initNodeStoreRootDirectory(Configuration conf) throws IOException {
// mkdir of root dir path with retry logic // mkdir of root dir path with retry logic
int maxRetries = conf.getInt(YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_RETRIES, int maxRetries = conf.getInt(YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_RETRIES,
YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_DEFAULT_RETRIES); YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_DEFAULT_RETRIES);
@ -73,11 +79,7 @@ protected void initStore(Configuration conf, Path fsStorePath,
while (!success && retryCount <= maxRetries) { while (!success && retryCount <= maxRetries) {
try { try {
if (!fs.exists(fsWorkingPath)) { success = fs.mkdirs(fsWorkingPath);
success = fs.mkdirs(fsWorkingPath);
} else {
success = true;
}
} catch (IOException e) { } catch (IOException e) {
retryCount++; retryCount++;
if (retryCount > maxRetries) { if (retryCount > maxRetries) {
@ -91,8 +93,6 @@ protected void initStore(Configuration conf, Path fsStorePath,
} }
} }
} }
this.replication = conf.getInt(YarnConfiguration.FS_STORE_FILE_REPLICATION,
YarnConfiguration.DEFAULT_FS_STORE_FILE_REPLICATION);
LOG.info("Created store directory :" + fsWorkingPath); LOG.info("Created store directory :" + fsWorkingPath);
} }

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
View File

@ -24,6 +24,7 @@
import java.util.Collection; import java.util.Collection;
import java.util.Map; import java.util.Map;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.Timeout;
@ -348,27 +349,13 @@ void testSerilizationAfterRecovery(String className) throws Exception {
@MethodSource("getParameters") @MethodSource("getParameters")
@ParameterizedTest @ParameterizedTest
void testRootMkdirOnInitStoreWhenRootDirectoryAlreadyExists(String className) throws Exception { void testRootMkdirOnInitStore(String className) throws Exception {
initTestFileSystemNodeLabelsStore(className);
final FileSystem mockFs = Mockito.mock(FileSystem.class);
final FileSystemNodeLabelsStore mockStore = createMockNodeLabelsStore(mockFs);
final int expectedMkdirsCount = 0;
Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
.thenReturn(true);
verifyMkdirsCount(mockStore, expectedMkdirsCount);
}
@MethodSource("getParameters")
@ParameterizedTest
void testRootMkdirOnInitStoreWhenRootDirectoryNotExists(String className) throws Exception {
initTestFileSystemNodeLabelsStore(className); initTestFileSystemNodeLabelsStore(className);
final FileSystem mockFs = Mockito.mock(FileSystem.class); final FileSystem mockFs = Mockito.mock(FileSystem.class);
final FileSystemNodeLabelsStore mockStore = createMockNodeLabelsStore(mockFs); final FileSystemNodeLabelsStore mockStore = createMockNodeLabelsStore(mockFs);
final int expectedMkdirsCount = 1; final int expectedMkdirsCount = 1;
Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class))) Mockito.when(mockStore.getFs().mkdirs(Mockito.any(Path.class))).thenReturn(true);
.thenReturn(false).thenReturn(true);
verifyMkdirsCount(mockStore, expectedMkdirsCount); verifyMkdirsCount(mockStore, expectedMkdirsCount);
} }
@ -378,10 +365,11 @@ void testRootMkdirOnInitStoreRetryLogic(String className) throws Exception {
initTestFileSystemNodeLabelsStore(className); initTestFileSystemNodeLabelsStore(className);
final FileSystem mockFs = Mockito.mock(FileSystem.class); final FileSystem mockFs = Mockito.mock(FileSystem.class);
final FileSystemNodeLabelsStore mockStore = createMockNodeLabelsStore(mockFs); final FileSystemNodeLabelsStore mockStore = createMockNodeLabelsStore(mockFs);
final int expectedMkdirsCount = 2; final int expectedMkdirsCount = 3;
Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class))) Mockito.when(mockStore.getFs().mkdirs(Mockito.any(Path.class)))
.thenReturn(false).thenReturn(false).thenReturn(true); .thenThrow(SafeModeException.class).thenThrow(SafeModeException.class)
.thenReturn(true);
verifyMkdirsCount(mockStore, expectedMkdirsCount); verifyMkdirsCount(mockStore, expectedMkdirsCount);
} }