diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 6be1efbc88..80b8b37d52 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -344,6 +344,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-3013. Removed YarnConfiguration.YARN_SECURITY_INFO and its usage as it doesn't affect security any more. (vinodkv) + MAPREDUCE-2907. Changed log level for various messages in ResourceManager + from INFO to DEBUG. (Ravi Prakash via vinodkv) + OPTIMIZATIONS MAPREDUCE-2026. Make JobTracker.getJobCounters() and diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index e3868c3c4d..4a47462a2d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -991,10 +991,9 @@ protected void setup(JobImpl job) throws IOException { String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path path = MRApps.getStagingAreaDir(job.conf, user); - LOG.info("DEBUG --- startJobs:" - + " parent=" - + path + " child=" - + oldJobIDString); + if(LOG.isDebugEnabled()) { + LOG.debug("startJobs: parent=" + path + " child=" + oldJobIDString); + } job.remoteJobSubmitDir = FileSystem.get(job.conf).makeQualified( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 180ecbe9d6..5bc171141d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -258,7 +258,7 @@ public String getStagingAreaDir() throws IOException, InterruptedException { String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path path = MRApps.getStagingAreaDir(conf, user); - LOG.info("DEBUG --- getStagingAreaDir: dir=" + path); + LOG.debug("getStagingAreaDir: dir=" + path); return path.toString(); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 41b60adba5..f0c38db2fa 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -464,21 +464,25 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, if (!ask.isEmpty()) { - LOG.info("DEBUG --- allocate: pre-update" + + if(LOG.isDebugEnabled()) { + LOG.debug("allocate: pre-update" + " applicationAttemptId=" + applicationAttemptId + " application=" + application); + } application.showRequests(); // Update application requests application.updateResourceRequests(ask); - LOG.info("DEBUG --- allocate: post-update"); + LOG.debug("allocate: post-update"); application.showRequests(); } - LOG.info("DEBUG --- allocate:" + + if(LOG.isDebugEnabled()) { + LOG.debug("allocate:" + " applicationAttemptId=" + applicationAttemptId + " #ask=" + ask.size()); + } return new Allocation( application.pullNewlyAllocatedContainers(), @@ -547,14 +551,16 @@ private synchronized void nodeUpdate(RMNode nm, // Process completed containers for (ContainerStatus completedContainer : completedContainers) { ContainerId containerId = completedContainer.getContainerId(); - LOG.info("DEBUG --- Container FINISHED: " + containerId); + LOG.debug("Container FINISHED: " + containerId); completedContainer(getRMContainer(containerId), completedContainer, RMContainerEventType.FINISHED); } // Now node data structures are upto date and ready for scheduling. - LOG.info("DEBUG -- Node being looked for scheduling " + nm + if(LOG.isDebugEnabled()) { + LOG.debug("Node being looked for scheduling " + nm + " availableResource: " + node.getAvailableResource()); + } // Assign new containers... // 1. Check for reserved applications diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 24754e4e75..db597447a8 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -184,9 +184,10 @@ public LeafQueue(CapacitySchedulerContext cs, maxActiveApplications, maxActiveApplicationsPerUser, state, acls); - LOG.info("DEBUG --- LeafQueue:" + - " name=" + queueName + - ", fullname=" + getQueuePath()); + if(LOG.isDebugEnabled()) { + LOG.debug("LeafQueue:" + " name=" + queueName + + ", fullname=" + getQueuePath()); + } this.pendingApplications = new TreeSet(applicationComparator); @@ -673,9 +674,10 @@ private synchronized SchedulerApp getApplication( public synchronized Resource assignContainers(Resource clusterResource, SchedulerNode node) { - LOG.info("DEBUG --- assignContainers:" + - " node=" + node.getHostName() + - " #applications=" + activeApplications.size()); + if(LOG.isDebugEnabled()) { + LOG.debug("assignContainers: node=" + node.getHostName() + + " #applications=" + activeApplications.size()); + } // Check for reserved resources RMContainer reservedContainer = node.getReservedContainer(); @@ -689,8 +691,10 @@ private synchronized SchedulerApp getApplication( // Try to assign containers to applications in order for (SchedulerApp application : activeApplications) { - LOG.info("DEBUG --- pre-assignContainers for application " - + application.getApplicationId()); + if(LOG.isDebugEnabled()) { + LOG.debug("pre-assignContainers for application " + + application.getApplicationId()); + } application.showRequests(); synchronized (application) { @@ -750,8 +754,10 @@ private synchronized SchedulerApp getApplication( } } - LOG.info("DEBUG --- post-assignContainers for application " + if(LOG.isDebugEnabled()) { + LOG.debug("post-assignContainers for application " + application.getApplicationId()); + } application.showRequests(); } @@ -1094,11 +1100,10 @@ private Resource assignContainer(Resource clusterResource, SchedulerNode node, SchedulerApp application, Priority priority, ResourceRequest request, NodeType type, RMContainer rmContainer) { if (LOG.isDebugEnabled()) { - LOG.info("DEBUG --- assignContainers:" + - " node=" + node.getHostName() + - " application=" + application.getApplicationId().getId() + - " priority=" + priority.getPriority() + - " request=" + request + " type=" + type); + LOG.debug("assignContainers: node=" + node.getHostName() + + " application=" + application.getApplicationId().getId() + + " priority=" + priority.getPriority() + + " request=" + request + " type=" + type); } Resource capability = request.getCapability(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 84d315417a..dc363a210f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -197,7 +197,9 @@ void setChildQueues(Collection childQueues) { this.childQueues.clear(); this.childQueues.addAll(childQueues); - LOG.info("DEBUG --- setChildQueues: " + getChildQueuesToPrint()); + if (LOG.isDebugEnabled()) { + LOG.debug("setChildQueues: " + getChildQueuesToPrint()); + } } @Override @@ -497,8 +499,10 @@ public synchronized Resource assignContainers( Resource assigned = Resources.createResource(0); while (canAssign(node)) { - LOG.info("DEBUG --- Trying to assign containers to child-queue of " + - getQueueName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to assign containers to child-queue of " + + getQueueName()); + } // Are we over maximum-capacity for this queue? if (!assignToQueue(clusterResource)) { @@ -527,11 +531,12 @@ public synchronized Resource assignContainers( break; } - LOG.info("DEBUG ---" + - " parentQ=" + getQueueName() + - " assignedSoFarInThisIteration=" + assigned + - " utilization=" + getUtilization()); - + if (LOG.isDebugEnabled()) { + LOG.debug("ParentQ=" + getQueueName() + + " assignedSoFarInThisIteration=" + assigned + + " utilization=" + getUtilization()); + } + // Do not assign more than one container if this isn't the root queue if (!rootQueue) { break; @@ -571,13 +576,15 @@ synchronized Resource assignContainersToChildQueues(Resource cluster, // Try to assign to most 'under-served' sub-queue for (Iterator iter=childQueues.iterator(); iter.hasNext();) { CSQueue childQueue = iter.next(); - LOG.info("DEBUG --- Trying to assign to" + - " queue: " + childQueue.getQueuePath() + - " stats: " + childQueue); + if(LOG.isDebugEnabled()) { + LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath() + + " stats: " + childQueue); + } assigned = childQueue.assignContainers(cluster, node); - LOG.info("DEBUG --- Assignedto" + - " queue: " + childQueue.getQueuePath() + - " stats: " + childQueue + " --> " + assigned.getMemory()); + if(LOG.isDebugEnabled()) { + LOG.debug("Assignedto queue: " + childQueue.getQueuePath() + + " stats: " + childQueue + " --> " + assigned.getMemory()); + } // If we do assign, remove the queue and re-insert in-order to re-sort if (Resources.greaterThan(assigned, Resources.none())) { @@ -602,8 +609,10 @@ String getChildQueuesToPrint() { return sb.toString(); } void printChildQueues() { - LOG.info("DEBUG --- printChildQueues - queue: " + getQueuePath() + - " child-queues: " + getChildQueuesToPrint()); + if (LOG.isDebugEnabled()) { + LOG.debug("printChildQueues - queue: " + getQueuePath() + + " child-queues: " + getChildQueuesToPrint()); + } } @Override diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index b031504eef..beb1dd3f60 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -595,7 +595,7 @@ private synchronized void nodeUpdate(RMNode rmNode, // Process completed containers for (ContainerStatus completedContainer : completedContainers) { ContainerId containerId = completedContainer.getContainerId(); - LOG.info("DEBUG --- Container FINISHED: " + containerId); + LOG.debug("Container FINISHED: " + containerId); containerCompleted(getRMContainer(containerId), completedContainer, RMContainerEventType.FINISHED); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 423b519fec..ee6cc99f0b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -164,9 +164,10 @@ public synchronized void addTask(Task task) { if (requests == null) { requests = new HashMap(); this.requests.put(priority, requests); - LOG.info("DEBUG --- Added" + - " priority=" + priority + - " application=" + applicationId); + if(LOG.isDebugEnabled()) { + LOG.debug("Added priority=" + priority + " application=" + + applicationId); + } } final Resource capability = requestSpec.get(priority); @@ -182,9 +183,10 @@ public synchronized void addTask(Task task) { LOG.info("Added task " + task.getTaskId() + " to application " + applicationId + " at priority " + priority); - LOG.info("DEBUG --- addTask:" + - " application=" + applicationId + - " #asks=" + ask.size()); + if(LOG.isDebugEnabled()) { + LOG.debug("addTask: application=" + applicationId + + " #asks=" + ask.size()); + } // Create resource requests for (String host : task.getHosts()) { @@ -245,23 +247,24 @@ private synchronized void addResourceRequest( org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest( request)); // clone to ensure the RM doesn't manipulate the same obj - LOG.info("DEBUG --- addResourceRequest:" + - " applicationId=" + applicationId.getId() + - " priority=" + priority.getPriority() + - " resourceName=" + resourceName + - " capability=" + capability + - " numContainers=" + request.getNumContainers() + - " #asks=" + ask.size()); + if(LOG.isDebugEnabled()) { + LOG.debug("addResourceRequest: applicationId=" + applicationId.getId() + + " priority=" + priority.getPriority() + + " resourceName=" + resourceName + " capability=" + capability + + " numContainers=" + request.getNumContainers() + + " #asks=" + ask.size()); + } } public synchronized List getResources() throws IOException { - LOG.info("DEBUG --- getResources begin:" + - " application=" + applicationId + - " #ask=" + ask.size()); - for (ResourceRequest request : ask) { - LOG.info("DEBUG --- getResources:" + - " application=" + applicationId + - " ask-request=" + request); + if(LOG.isDebugEnabled()) { + LOG.debug("getResources begin:" + " application=" + applicationId + + " #ask=" + ask.size()); + + for (ResourceRequest request : ask) { + LOG.debug("getResources:" + " application=" + applicationId + + " ask-request=" + request); + } } // Get resources from the ResourceManager @@ -280,9 +283,10 @@ public synchronized List getResources() throws IOException { // Clear state for next interaction with ResourceManager ask.clear(); - LOG.info("DEBUG --- getResources() for " + applicationId + ":" + - " ask=" + ask.size() + - " recieved=" + containers.size()); + if(LOG.isDebugEnabled()) { + LOG.debug("getResources() for " + applicationId + ":" + + " ask=" + ask.size() + " recieved=" + containers.size()); + } return containers; } @@ -353,22 +357,22 @@ private void updateResourceRequests(Map requests, NodeType type, Task task) { if (type == NodeType.NODE_LOCAL) { for (String host : task.getHosts()) { - LOG.info("DEBUG --- updateResourceRequests:" + - " application=" + applicationId + - " type=" + type + - " host=" + host + - " request=" + ((requests == null) ? "null" : requests.get(host))); + if(LOG.isDebugEnabled()) { + LOG.debug("updateResourceRequests:" + " application=" + applicationId + + " type=" + type + " host=" + host + + " request=" + ((requests == null) ? "null" : requests.get(host))); + } updateResourceRequest(requests.get(host)); } } if (type == NodeType.NODE_LOCAL || type == NodeType.RACK_LOCAL) { for (String rack : task.getRacks()) { - LOG.info("DEBUG --- updateResourceRequests:" + - " application=" + applicationId + - " type=" + type + - " rack=" + rack + - " request=" + ((requests == null) ? "null" : requests.get(rack))); + if(LOG.isDebugEnabled()) { + LOG.debug("updateResourceRequests:" + " application=" + applicationId + + " type=" + type + " rack=" + rack + + " request=" + ((requests == null) ? "null" : requests.get(rack))); + } updateResourceRequest(requests.get(rack)); } } @@ -378,9 +382,10 @@ private void updateResourceRequests(Map requests, org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY) ); - LOG.info("DEBUG --- updateResourceRequests:" + - " application=" + applicationId + - " #asks=" + ask.size()); + if(LOG.isDebugEnabled()) { + LOG.debug("updateResourceRequests:" + " application=" + applicationId + + " #asks=" + ask.size()); + } } private void updateResourceRequest(ResourceRequest request) { @@ -392,9 +397,10 @@ private void updateResourceRequest(ResourceRequest request) { org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest( request)); // clone to ensure the RM doesn't manipulate the same obj - LOG.info("DEBUG --- updateResourceRequest:" + - " application=" + applicationId + - " request=" + request); + if(LOG.isDebugEnabled()) { + LOG.debug("updateResourceRequest:" + " application=" + applicationId + + " request=" + request); + } } private ContainerLaunchContext createCLC(Container container) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 72ade5c1da..254ea0ad5a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -194,12 +194,11 @@ synchronized public StartContainerResponse startContainer( Resources.subtractFrom(available, containerLaunchContext.getResource()); Resources.addTo(used, containerLaunchContext.getResource()); - LOG.info("DEBUG --- startContainer:" + - " node=" + containerManagerAddress + - " application=" + applicationId + - " container=" + container + - " available=" + available + - " used=" + used); + if(LOG.isDebugEnabled()) { + LOG.debug("startContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + container + + " available=" + available + " used=" + used); + } StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class); return response; @@ -254,12 +253,11 @@ synchronized public StopContainerResponse stopContainer(StopContainerRequest req Resources.addTo(available, container.getResource()); Resources.subtractFrom(used, container.getResource()); - LOG.info("DEBUG --- stopContainer:" + - " node=" + containerManagerAddress + - " application=" + applicationId + - " container=" + containerID + - " available=" + available + - " used=" + used); + if(LOG.isDebugEnabled()) { + LOG.debug("stopContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + containerID + + " available=" + available + " used=" + used); + } StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class); return response;