MAPREDUCE-2907. Changed log level for various messages in ResourceManager from INFO to DEBUG. Contributed by Ravi Prakash.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1179178 13f79535-47bb-0310-9956-ffa450edef68
2011-10-05 11:56:54 +00:00 · 2011-10-05 11:56:54 +00:00 · f24dcb3449
commit f24dcb3449
parent 66137cf17c
9 changed files with 118 additions and 92 deletions
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@ -344,6 +344,9 @@ Release 0.23.0 - Unreleased
    MAPREDUCE-3013. Removed YarnConfiguration.YARN_SECURITY_INFO and its usage
    as it doesn't affect security any more. (vinodkv)

+    MAPREDUCE-2907. Changed log level for various messages in ResourceManager
+    from INFO to DEBUG. (Ravi Prakash via vinodkv)
+
  OPTIMIZATIONS

    MAPREDUCE-2026. Make JobTracker.getJobCounters() and
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@ -991,10 +991,9 @@ protected void setup(JobImpl job) throws IOException {
      String user = 
        UserGroupInformation.getCurrentUser().getShortUserName();
      Path path = MRApps.getStagingAreaDir(job.conf, user);
-      LOG.info("DEBUG --- startJobs:"
-          + " parent="
-          + path + " child="
-          + oldJobIDString);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("startJobs: parent=" + path + " child=" + oldJobIDString);
+      }

      job.remoteJobSubmitDir =
          FileSystem.get(job.conf).makeQualified(
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@ -258,7 +258,7 @@ public String getStagingAreaDir() throws IOException, InterruptedException {
    String user = 
      UserGroupInformation.getCurrentUser().getShortUserName();
    Path path = MRApps.getStagingAreaDir(conf, user);
-    LOG.info("DEBUG --- getStagingAreaDir: dir=" + path);
+    LOG.debug("getStagingAreaDir: dir=" + path);
    return path.toString();
  }

--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@ -464,21 +464,25 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,

      if (!ask.isEmpty()) {

-        LOG.info("DEBUG --- allocate: pre-update" +
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("allocate: pre-update" +
            " applicationAttemptId=" + applicationAttemptId + 
            " application=" + application);
+        }
        application.showRequests();
  
        // Update application requests
        application.updateResourceRequests(ask);
  
-        LOG.info("DEBUG --- allocate: post-update");
+        LOG.debug("allocate: post-update");
        application.showRequests();
      }

-      LOG.info("DEBUG --- allocate:" +
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("allocate:" +
          " applicationAttemptId=" + applicationAttemptId + 
          " #ask=" + ask.size());
+      }

      return new Allocation(
          application.pullNewlyAllocatedContainers(), 
@ -547,14 +551,16 @@ private synchronized void nodeUpdate(RMNode nm,
    // Process completed containers
    for (ContainerStatus completedContainer : completedContainers) {
      ContainerId containerId = completedContainer.getContainerId();
-      LOG.info("DEBUG --- Container FINISHED: " + containerId);
+      LOG.debug("Container FINISHED: " + containerId);
      completedContainer(getRMContainer(containerId), 
          completedContainer, RMContainerEventType.FINISHED);
    }

    // Now node data structures are upto date and ready for scheduling.
-    LOG.info("DEBUG -- Node being looked for scheduling " + nm
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Node being looked for scheduling " + nm
        + " availableResource: " + node.getAvailableResource());
+    }

    // Assign new containers...
    // 1. Check for reserved applications
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@ -184,9 +184,10 @@ public LeafQueue(CapacitySchedulerContext cs,
        maxActiveApplications, maxActiveApplicationsPerUser,
        state, acls);

-    LOG.info("DEBUG --- LeafQueue:" +
-        " name=" + queueName + 
-        ", fullname=" + getQueuePath());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("LeafQueue:" + " name=" + queueName
+        + ", fullname=" + getQueuePath());
+    }

    this.pendingApplications = 
        new TreeSet<SchedulerApp>(applicationComparator);
@ -673,9 +674,10 @@ private synchronized SchedulerApp getApplication(
  public synchronized Resource 
  assignContainers(Resource clusterResource, SchedulerNode node) {

-    LOG.info("DEBUG --- assignContainers:" +
-        " node=" + node.getHostName() + 
-        " #applications=" + activeApplications.size());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("assignContainers: node=" + node.getHostName()
+        + " #applications=" + activeApplications.size());
+    }
    
    // Check for reserved resources
    RMContainer reservedContainer = node.getReservedContainer();
@ -689,8 +691,10 @@ private synchronized SchedulerApp getApplication(
    // Try to assign containers to applications in order
    for (SchedulerApp application : activeApplications) {
      
-      LOG.info("DEBUG --- pre-assignContainers for application "
-          + application.getApplicationId());
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("pre-assignContainers for application "
+        + application.getApplicationId());
+      }
      application.showRequests();

      synchronized (application) {
@ -750,8 +754,10 @@ private synchronized SchedulerApp getApplication(
        }
      }

-      LOG.info("DEBUG --- post-assignContainers for application "
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("post-assignContainers for application "
          + application.getApplicationId());
+      }
      application.showRequests();
    }
  
@ -1094,11 +1100,10 @@ private Resource assignContainer(Resource clusterResource, SchedulerNode node,
      SchedulerApp application, Priority priority, 
      ResourceRequest request, NodeType type, RMContainer rmContainer) {
    if (LOG.isDebugEnabled()) {
-      LOG.info("DEBUG --- assignContainers:" +
-          " node=" + node.getHostName() + 
-          " application=" + application.getApplicationId().getId() + 
-          " priority=" + priority.getPriority() + 
-          " request=" + request + " type=" + type);
+      LOG.debug("assignContainers: node=" + node.getHostName()
+        + " application=" + application.getApplicationId().getId()
+        + " priority=" + priority.getPriority()
+        + " request=" + request + " type=" + type);
    }
    Resource capability = request.getCapability();

--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@ -197,7 +197,9 @@ void setChildQueues(Collection<CSQueue> childQueues) {
    
    this.childQueues.clear();
    this.childQueues.addAll(childQueues);
-    LOG.info("DEBUG --- setChildQueues: " + getChildQueuesToPrint());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("setChildQueues: " + getChildQueuesToPrint());
+    }
  }
  
  @Override
@ -497,8 +499,10 @@ public synchronized Resource assignContainers(
    Resource assigned = Resources.createResource(0);

    while (canAssign(node)) {
-      LOG.info("DEBUG --- Trying to assign containers to child-queue of " + 
-          getQueueName());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Trying to assign containers to child-queue of "
+          + getQueueName());
+      }
      
      // Are we over maximum-capacity for this queue?
      if (!assignToQueue(clusterResource)) {
@ -527,11 +531,12 @@ public synchronized Resource assignContainers(
        break;
      }

-      LOG.info("DEBUG ---" +
-      		" parentQ=" + getQueueName() + 
-      		" assignedSoFarInThisIteration=" + assigned + 
-      		" utilization=" + getUtilization());
-      
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("ParentQ=" + getQueueName()
+          + " assignedSoFarInThisIteration=" + assigned
+          + " utilization=" + getUtilization());
+      }
+
      // Do not assign more than one container if this isn't the root queue
      if (!rootQueue) {
        break;
@ -571,13 +576,15 @@ synchronized Resource assignContainersToChildQueues(Resource cluster,
    // Try to assign to most 'under-served' sub-queue
    for (Iterator<CSQueue> iter=childQueues.iterator(); iter.hasNext();) {
      CSQueue childQueue = iter.next();
-      LOG.info("DEBUG --- Trying to assign to" +
-      		" queue: " + childQueue.getQueuePath() + 
-      		" stats: " + childQueue);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
+          + " stats: " + childQueue);
+      }
      assigned = childQueue.assignContainers(cluster, node);
-      LOG.info("DEBUG --- Assignedto" +
-          " queue: " + childQueue.getQueuePath() + 
-          " stats: " + childQueue + " --> " + assigned.getMemory());
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Assignedto queue: " + childQueue.getQueuePath()
+          + " stats: " + childQueue + " --> " + assigned.getMemory());
+      }

      // If we do assign, remove the queue and re-insert in-order to re-sort
      if (Resources.greaterThan(assigned, Resources.none())) {
@ -602,8 +609,10 @@ String getChildQueuesToPrint() {
    return sb.toString();
  }
  void printChildQueues() {
-    LOG.info("DEBUG --- printChildQueues - queue: " + getQueuePath() + 
-        " child-queues: " + getChildQueuesToPrint());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("printChildQueues - queue: " + getQueuePath()
+        + " child-queues: " + getChildQueuesToPrint());
+    }
  }
  
  @Override
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@ -595,7 +595,7 @@ private synchronized void nodeUpdate(RMNode rmNode,
    // Process completed containers
    for (ContainerStatus completedContainer : completedContainers) {
      ContainerId containerId = completedContainer.getContainerId();
-      LOG.info("DEBUG --- Container FINISHED: " + containerId);
+      LOG.debug("Container FINISHED: " + containerId);
      containerCompleted(getRMContainer(containerId), 
          completedContainer, RMContainerEventType.FINISHED);
    }
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
@ -164,9 +164,10 @@ public synchronized void addTask(Task task) {
    if (requests == null) {
      requests = new HashMap<String, ResourceRequest>();
      this.requests.put(priority, requests);
-      LOG.info("DEBUG --- Added" +
-      		" priority=" + priority + 
-      		" application=" + applicationId);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Added priority=" + priority + " application="
+          + applicationId);
+      }
    }
    
    final Resource capability = requestSpec.get(priority);
@ -182,9 +183,10 @@ public synchronized void addTask(Task task) {
    LOG.info("Added task " + task.getTaskId() + " to application " + 
        applicationId + " at priority " + priority);
    
-    LOG.info("DEBUG --- addTask:" +
-    		" application=" + applicationId + 
-    		" #asks=" + ask.size());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("addTask: application=" + applicationId
+        + " #asks=" + ask.size());
+    }
    
    // Create resource requests
    for (String host : task.getHosts()) {
@ -245,23 +247,24 @@ private synchronized void addResourceRequest(
        org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest(
            request)); // clone to ensure the RM doesn't manipulate the same obj
    
-    LOG.info("DEBUG --- addResourceRequest:" +
-    		" applicationId=" + applicationId.getId() +
-    		" priority=" + priority.getPriority() + 
-        " resourceName=" + resourceName + 
-        " capability=" + capability +
-        " numContainers=" + request.getNumContainers() + 
-        " #asks=" + ask.size());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("addResourceRequest: applicationId=" + applicationId.getId()
+        + " priority=" + priority.getPriority()
+        + " resourceName=" + resourceName + " capability=" + capability
+        + " numContainers=" + request.getNumContainers()
+        + " #asks=" + ask.size());
+    }
  }
  
  public synchronized List<Container> getResources() throws IOException {
-    LOG.info("DEBUG --- getResources begin:" +
-        " application=" + applicationId + 
-        " #ask=" + ask.size());
-    for (ResourceRequest request : ask) {
-      LOG.info("DEBUG --- getResources:" +
-          " application=" + applicationId + 
-          " ask-request=" + request);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("getResources begin:" + " application=" + applicationId
+        + " #ask=" + ask.size());
+
+      for (ResourceRequest request : ask) {
+        LOG.debug("getResources:" + " application=" + applicationId
+          + " ask-request=" + request);
+      }
    }
    
    // Get resources from the ResourceManager
@ -280,9 +283,10 @@ public synchronized List<Container> getResources() throws IOException {
    // Clear state for next interaction with ResourceManager
    ask.clear();
    
-    LOG.info("DEBUG --- getResources() for " + applicationId + ":" +
-    		" ask=" + ask.size() + 
-    		" recieved=" + containers.size());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("getResources() for " + applicationId + ":"
+        + " ask=" + ask.size() + " recieved=" + containers.size());
+    }
    
    return containers;
  }
@ -353,22 +357,22 @@ private void updateResourceRequests(Map<String, ResourceRequest> requests,
      NodeType type, Task task) {
    if (type == NodeType.NODE_LOCAL) {
      for (String host : task.getHosts()) {
-        LOG.info("DEBUG --- updateResourceRequests:" +
-            " application=" + applicationId +
-        		" type=" + type + 
-        		" host=" + host + 
-        		" request=" + ((requests == null) ? "null" : requests.get(host)));
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("updateResourceRequests:" + " application=" + applicationId
+            + " type=" + type + " host=" + host
+            + " request=" + ((requests == null) ? "null" : requests.get(host)));
+        }
        updateResourceRequest(requests.get(host));
      }
    }
    
    if (type == NodeType.NODE_LOCAL || type == NodeType.RACK_LOCAL) {
      for (String rack : task.getRacks()) {
-        LOG.info("DEBUG --- updateResourceRequests:" +
-            " application=" + applicationId +
-            " type=" + type + 
-            " rack=" + rack + 
-            " request=" + ((requests == null) ? "null" : requests.get(rack)));
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("updateResourceRequests:" + " application=" + applicationId
+            + " type=" + type + " rack=" + rack
+            + " request=" + ((requests == null) ? "null" : requests.get(rack)));
+        }
        updateResourceRequest(requests.get(rack));
      }
    }
@ -378,9 +382,10 @@ private void updateResourceRequests(Map<String, ResourceRequest> requests,
            org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY)
            );
    
-    LOG.info("DEBUG --- updateResourceRequests:" +
-        " application=" + applicationId +
-    		" #asks=" + ask.size());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("updateResourceRequests:" + " application=" + applicationId
+        + " #asks=" + ask.size());
+    }
  }
  
  private void updateResourceRequest(ResourceRequest request) {
@ -392,9 +397,10 @@ private void updateResourceRequest(ResourceRequest request) {
        org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest(
        request)); // clone to ensure the RM doesn't manipulate the same obj

-    LOG.info("DEBUG --- updateResourceRequest:" +
-        " application=" + applicationId +
-    		" request=" + request);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("updateResourceRequest:" + " application=" + applicationId
+        + " request=" + request);
+    }
  }

  private ContainerLaunchContext createCLC(Container container) {
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@ -194,12 +194,11 @@ synchronized public StartContainerResponse startContainer(
    Resources.subtractFrom(available, containerLaunchContext.getResource());
    Resources.addTo(used, containerLaunchContext.getResource());
    
-    LOG.info("DEBUG --- startContainer:" +
-        " node=" + containerManagerAddress +
-        " application=" + applicationId + 
-        " container=" + container +
-        " available=" + available +
-        " used=" + used);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("startContainer:" + " node=" + containerManagerAddress
+        + " application=" + applicationId + " container=" + container
+        + " available=" + available + " used=" + used);
+    }

    StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class);
    return response;
@ -254,12 +253,11 @@ synchronized public StopContainerResponse stopContainer(StopContainerRequest req
    Resources.addTo(available, container.getResource());
    Resources.subtractFrom(used, container.getResource());

-    LOG.info("DEBUG --- stopContainer:" +
-        " node=" + containerManagerAddress +
-        " application=" + applicationId + 
-        " container=" + containerID +
-        " available=" + available +
-        " used=" + used);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("stopContainer:" + " node=" + containerManagerAddress
+        + " application=" + applicationId + " container=" + containerID
+        + " available=" + available + " used=" + used);
+    }

    StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class);
    return response;