big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10884: Handle empty owners to parse log files (#3318)

Browse Source 
* YARN 10884 : Parse log files which has empty owner

* Removing the whitespace

* Added a test case for null user

* Fixed indentations

* Fixed the indentation for test cases

Co-authored-by: Swathi C <cswathi@microsoft.com>
This commit is contained in:
Swathi Chandrashekar 2021-09-07 21:37:51 +05:30 committed by GitHub
parent 6e3aeb1544
commit 9512c774c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 34 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
View File

@ -134,6 +134,7 @@ public class EntityGroupFSTimelineStore extends CompositeService
private int appCacheMaxSize = 0;
private List<TimelineEntityGroupPlugin> cacheIdPlugins;
private Map<TimelineEntityGroupId, EntityCacheItem> cachedLogs;
private boolean aclsEnabled;
@VisibleForTesting
@InterfaceAudience.Private
@ -204,6 +205,8 @@ protected boolean removeEldestEntry(
YarnConfiguration
.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT));
fs = activeRootPath.getFileSystem(conf);
aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
YarnConfiguration.DEFAULT_YARN_ACL_ENABLE);
CallerContext.setCurrent(
new CallerContext.Builder(ATS_V15_SERVER_DFS_CALLER_CTXT).build());
super.serviceInit(conf);
@ -766,16 +769,24 @@ long scanForLogs() throws IOException {
continue;
}
String filename = statCache.getPath().getName();
String owner = statCache.getOwner();
//YARN-10884:Owner of File is set to Null on WASB Append Operation.ATS fails to read such
//files as UGI cannot be constructed using Null User.To Fix this,anonymous user is set
//when ACL us Disabled as the UGI is not needed there
if ((owner == null || owner.isEmpty()) && !aclsEnabled) {
LOG.debug("The owner was null when acl disabled, hence making the owner anonymous");
owner = "anonymous";
}
// We should only update time for log files.
boolean shouldSetTime = true;
LOG.debug("scan for log file: {}", filename);
if (filename.startsWith(DOMAIN_LOG_PREFIX)) {
addSummaryLog(attemptDirName, filename, statCache.getOwner(), true);
addSummaryLog(attemptDirName, filename, owner, true);
} else if (filename.startsWith(SUMMARY_LOG_PREFIX)) {
addSummaryLog(attemptDirName, filename, statCache.getOwner(),
addSummaryLog(attemptDirName, filename, owner,
false);
} else if (filename.startsWith(ENTITY_LOG_PREFIX)) {
addDetailLog(attemptDirName, filename, statCache.getOwner());
addDetailLog(attemptDirName, filename, owner);
} else {
shouldSetTime = false;
}

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.fs.FileContextTestHelper;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
@ -67,6 +68,8 @@
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
@ -279,6 +282,23 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
assertEquals(beforeScan + 2L, scanned.value());
}
@Test
public void testWithAnonymousUser() throws Exception {
try {
TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
EntityGroupFSTimelineStore.AppLogs appLogs =
store.new AppLogs(mainTestAppId, mainTestAppDirPath,
AppState.COMPLETED);
FileStatus fileStatus = mock(FileStatus.class);
when(fileStatus.getOwner()).thenReturn(null);
appLogs.scanForLogs();
appLogs.parseSummaryLogs(tdm);
PluginStoreTestUtils.verifyTestEntities(tdm);
} catch (IllegalArgumentException ie) {
Assert.fail("No exception needs to be thrown as anonymous user is configured");
}
}
@Test
public void testCleanLogs() throws Exception {
// Create test dirs and files