YARN-5027. NM should clean up app log dirs after NM restart. Contributed by sandflee
This commit is contained in:
parent
c017171da0
commit
7146359bfd
@ -242,6 +242,7 @@ public void serviceInit(Configuration conf) throws Exception {
|
||||
|
||||
if (!stateStore.canRecover()|| stateStore.isNewlyCreated()) {
|
||||
cleanUpLocalDirs(lfs, delService);
|
||||
cleanupLogDirs(lfs, delService);
|
||||
initializeLocalDirs(lfs);
|
||||
initializeLogDirs(lfs);
|
||||
}
|
||||
@ -1374,9 +1375,9 @@ private void initializeLogDirs(FileContext lfs) {
|
||||
}
|
||||
}
|
||||
|
||||
private void initializeLogDir(FileContext lfs, String logDir) {
|
||||
private void initializeLogDir(FileContext fs, String logDir) {
|
||||
try {
|
||||
lfs.mkdir(new Path(logDir), null, true);
|
||||
fs.mkdir(new Path(logDir), null, true);
|
||||
} catch (FileAlreadyExistsException fe) {
|
||||
// do nothing
|
||||
} catch (IOException e) {
|
||||
@ -1386,6 +1387,57 @@ private void initializeLogDir(FileContext lfs, String logDir) {
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupLogDirs(FileContext fs, DeletionService del) {
|
||||
for (String logDir : dirsHandler.getLogDirsForCleanup()) {
|
||||
try {
|
||||
cleanupLogDir(fs, del, logDir);
|
||||
} catch (IOException e) {
|
||||
LOG.warn("failed to cleanup app log dir " + logDir, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupLogDir(FileContext fs, DeletionService del,
|
||||
String logDir) throws IOException {
|
||||
if (!fs.util().exists(new Path(logDir))){
|
||||
return;
|
||||
}
|
||||
renameAppLogDir(logDir);
|
||||
deleteAppLogDir(fs, del, logDir);
|
||||
}
|
||||
|
||||
private void renameAppLogDir(String logDir) throws IOException {
|
||||
long currentTimeStamp = System.currentTimeMillis();
|
||||
RemoteIterator<FileStatus> fileStatuses =
|
||||
lfs.listStatus(new Path(logDir));
|
||||
if (fileStatuses != null) {
|
||||
while (fileStatuses.hasNext()) {
|
||||
FileStatus fileStatus = fileStatuses.next();
|
||||
String appName = fileStatus.getPath().getName();
|
||||
if (appName.matches("^application_\\d+_\\d+$")) {
|
||||
lfs.rename(new Path(logDir, appName),
|
||||
new Path(logDir, appName + "_DEL_" + currentTimeStamp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteAppLogDir(FileContext fs, DeletionService del,
|
||||
String logDir) throws IOException {
|
||||
RemoteIterator<FileStatus> fileStatuses =
|
||||
fs.listStatus(new Path(logDir));
|
||||
if (fileStatuses != null) {
|
||||
while (fileStatuses.hasNext()) {
|
||||
FileStatus fileStatus = fileStatuses.next();
|
||||
String appName = fileStatus.getPath().getName();
|
||||
if (appName.matches("^application_\\d+_\\d+_DEL_\\d+$")) {
|
||||
LOG.info("delete app log dir," + appName);
|
||||
del.delete(null, fileStatus.getPath());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanUpLocalDirs(FileContext lfs, DeletionService del) {
|
||||
for (String localDir : dirsHandler.getLocalDirsForCleanup()) {
|
||||
cleanUpLocalDir(lfs, del, localDir);
|
||||
|
Loading…
Reference in New Issue
Block a user