From 9512c774c00c15e553bcf879a7abff338ce55580 Mon Sep 17 00:00:00 2001 From: Swathi Chandrashekar <85697049+swathic95@users.noreply.github.com> Date: Tue, 7 Sep 2021 21:37:51 +0530 Subject: [PATCH] YARN-10884: Handle empty owners to parse log files (#3318) * YARN 10884 : Parse log files which has empty owner * Removing the whitespace * Added a test case for null user * Fixed indentations * Fixed the indentation for test cases Co-authored-by: Swathi C --- .../timeline/EntityGroupFSTimelineStore.java | 17 +++++++++++++--- .../TestEntityGroupFSTimelineStore.java | 20 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index c2992b4e83..4e5b796a3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -134,6 +134,7 @@ public class EntityGroupFSTimelineStore extends CompositeService private int appCacheMaxSize = 0; private List cacheIdPlugins; private Map cachedLogs; + private boolean aclsEnabled; @VisibleForTesting @InterfaceAudience.Private @@ -204,6 +205,8 @@ protected boolean removeEldestEntry( YarnConfiguration .TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT)); fs = activeRootPath.getFileSystem(conf); + aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); CallerContext.setCurrent( new CallerContext.Builder(ATS_V15_SERVER_DFS_CALLER_CTXT).build()); super.serviceInit(conf); @@ -766,16 +769,24 @@ long scanForLogs() throws IOException { continue; } String filename = statCache.getPath().getName(); + String owner = statCache.getOwner(); + //YARN-10884:Owner of File is set to Null on WASB Append Operation.ATS fails to read such + //files as UGI cannot be constructed using Null User.To Fix this,anonymous user is set + //when ACL us Disabled as the UGI is not needed there + if ((owner == null || owner.isEmpty()) && !aclsEnabled) { + LOG.debug("The owner was null when acl disabled, hence making the owner anonymous"); + owner = "anonymous"; + } // We should only update time for log files. boolean shouldSetTime = true; LOG.debug("scan for log file: {}", filename); if (filename.startsWith(DOMAIN_LOG_PREFIX)) { - addSummaryLog(attemptDirName, filename, statCache.getOwner(), true); + addSummaryLog(attemptDirName, filename, owner, true); } else if (filename.startsWith(SUMMARY_LOG_PREFIX)) { - addSummaryLog(attemptDirName, filename, statCache.getOwner(), + addSummaryLog(attemptDirName, filename, owner, false); } else if (filename.startsWith(ENTITY_LOG_PREFIX)) { - addDetailLog(attemptDirName, filename, statCache.getOwner()); + addDetailLog(attemptDirName, filename, owner); } else { shouldSetTime = false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 7302ae130d..fdd47727c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileContextTestHelper; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.metrics2.lib.MutableCounterLong; @@ -67,6 +68,8 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils { @@ -279,6 +282,23 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, assertEquals(beforeScan + 2L, scanned.value()); } + @Test + public void testWithAnonymousUser() throws Exception { + try { + TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); + EntityGroupFSTimelineStore.AppLogs appLogs = + store.new AppLogs(mainTestAppId, mainTestAppDirPath, + AppState.COMPLETED); + FileStatus fileStatus = mock(FileStatus.class); + when(fileStatus.getOwner()).thenReturn(null); + appLogs.scanForLogs(); + appLogs.parseSummaryLogs(tdm); + PluginStoreTestUtils.verifyTestEntities(tdm); + } catch (IllegalArgumentException ie) { + Assert.fail("No exception needs to be thrown as anonymous user is configured"); + } + } + @Test public void testCleanLogs() throws Exception { // Create test dirs and files