Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke
This commit is contained in:
Szilard Nemeth
parent
364d38f00d
commit
843f66f4dc
@ -145,7 +145,8 @@ public AbstractCSQueue(CapacitySchedulerQueueContext queueContext, String queueN
|
||||
CSQueueMetrics metrics = old != null ?
|
||||
(CSQueueMetrics) old.getMetrics() :
|
||||
CSQueueMetrics.forQueue(getQueuePath(), parent,
|
||||
queueContext.getConfiguration().getEnableUserMetrics(), queueContext.getConfiguration());
|
||||
queueContext.getConfiguration().getEnableUserMetrics(),
|
||||
queueContext.getConfiguration());
|
||||
this.usageTracker = new CSQueueUsageTracker(metrics);
|
||||
|
||||
this.queueCapacities = new QueueCapacities(parent == null);
|
||||
|
||||
@ -453,8 +453,7 @@ public QueueInfo getQueueInfo(
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<QueueUserACLInfo>
|
||||
getQueueUserAclInfo(UserGroupInformation user) {
|
||||
public List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user) {
|
||||
readLock.lock();
|
||||
try {
|
||||
QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
|
||||
@ -527,8 +526,8 @@ public List<AppPriorityACLGroup> getPriorityACLs() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws
|
||||
IOException {
|
||||
public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
|
||||
throws IOException {
|
||||
|
||||
writeLock.lock();
|
||||
try {
|
||||
@ -634,7 +633,8 @@ public void validateSubmitApplication(ApplicationId applicationId,
|
||||
|
||||
// Check submission limits for queues
|
||||
//TODO recalculate max applications because they can depend on capacity
|
||||
if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) {
|
||||
if (getNumApplications() >= getMaxApplications() &&
|
||||
!(this instanceof AutoCreatedLeafQueue)) {
|
||||
String msg =
|
||||
"Queue " + getQueuePath() + " already has " + getNumApplications()
|
||||
+ " applications,"
|
||||
@ -646,7 +646,8 @@ public void validateSubmitApplication(ApplicationId applicationId,
|
||||
// Check submission limits for the user on this queue
|
||||
User user = usersManager.getUserAndAddIfAbsent(userName);
|
||||
//TODO recalculate max applications because they can depend on capacity
|
||||
if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) {
|
||||
if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&
|
||||
!(this instanceof AutoCreatedLeafQueue)) {
|
||||
String msg = "Queue " + getQueuePath() + " already has " + user
|
||||
.getTotalApplications() + " applications from user " + userName
|
||||
+ " cannot accept submission of application: " + applicationId;
|
||||
@ -825,10 +826,9 @@ protected void activateApplications() {
|
||||
calculateAndGetAMResourceLimitPerPartition(nodePartition);
|
||||
}
|
||||
|
||||
for (Iterator<FiCaSchedulerApp> fsApp =
|
||||
getPendingAppsOrderingPolicy()
|
||||
for (Iterator<FiCaSchedulerApp> fsApp = getPendingAppsOrderingPolicy()
|
||||
.getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR);
|
||||
fsApp.hasNext(); ) {
|
||||
fsApp.hasNext();) {
|
||||
FiCaSchedulerApp application = fsApp.next();
|
||||
ApplicationId applicationId = application.getApplicationId();
|
||||
|
||||
@ -864,7 +864,8 @@ protected void activateApplications() {
|
||||
+ " skipping enforcement to allow at least one application"
|
||||
+ " to start");
|
||||
} else{
|
||||
application.updateAMContainerDiagnostics(SchedulerApplicationAttempt.AMState.INACTIVATED,
|
||||
application.updateAMContainerDiagnostics(
|
||||
SchedulerApplicationAttempt.AMState.INACTIVATED,
|
||||
CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
|
||||
LOG.debug("Not activating application {} as amIfStarted: {}"
|
||||
+ " exceeds amLimit: {}", applicationId, amIfStarted, amLimit);
|
||||
@ -1189,9 +1190,8 @@ public CSAssignment assignContainers(Resource clusterResource,
|
||||
boolean needAssignToQueueCheck = true;
|
||||
IteratorSelector sel = new IteratorSelector();
|
||||
sel.setPartition(candidates.getPartition());
|
||||
for (Iterator<FiCaSchedulerApp> assignmentIterator =
|
||||
orderingPolicy.getAssignmentIterator(sel);
|
||||
assignmentIterator.hasNext(); ) {
|
||||
for (Iterator<FiCaSchedulerApp> assignmentIterator = orderingPolicy.getAssignmentIterator(sel);
|
||||
assignmentIterator.hasNext();) {
|
||||
FiCaSchedulerApp application = assignmentIterator.next();
|
||||
|
||||
ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
|
||||
@ -1821,13 +1821,8 @@ void allocateResource(Resource clusterResource,
|
||||
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
|
||||
RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
|
||||
RMNodeLabelsManager.NO_LABEL)) {
|
||||
TreeSet<RMContainer> rmContainers = null;
|
||||
if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(
|
||||
nodePartition))) {
|
||||
rmContainers = new TreeSet<>();
|
||||
ignorePartitionExclusivityRMContainers.put(nodePartition,
|
||||
rmContainers);
|
||||
}
|
||||
TreeSet<RMContainer> rmContainers = ignorePartitionExclusivityRMContainers.computeIfAbsent(
|
||||
nodePartition, k -> new TreeSet<>());
|
||||
rmContainers.add(rmContainer);
|
||||
}
|
||||
|
||||
@ -2195,8 +2190,7 @@ public void detachContainer(Resource clusterResource,
|
||||
* @return all ignored partition exclusivity RMContainers in the LeafQueue,
|
||||
* this will be used by preemption policy.
|
||||
*/
|
||||
public Map<String, TreeSet<RMContainer>>
|
||||
getIgnoreExclusivityRMContainers() {
|
||||
public Map<String, TreeSet<RMContainer>> getIgnoreExclusivityRMContainers() {
|
||||
Map<String, TreeSet<RMContainer>> clonedMap = new HashMap<>();
|
||||
|
||||
readLock.lock();
|
||||
|
||||
@ -26,7 +26,6 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractAutoCreatedLeafQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueueConfig;
|
||||
|
||||
@ -275,8 +275,9 @@ public void testLimitsComputation() throws Exception {
|
||||
Resource clusterResource =
|
||||
Resources.createResource(100 * 16 * GB, 100 * 16);
|
||||
|
||||
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB, 1),
|
||||
Resources.createResource(16*GB, 16), clusterResource);
|
||||
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
|
||||
Resources.createResource(GB, 1), Resources.createResource(16*GB, 16),
|
||||
clusterResource);
|
||||
CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager();
|
||||
CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
|
||||
|
||||
@ -299,7 +300,7 @@ public void testLimitsComputation() throws Exception {
|
||||
assertThat(queue.calculateAndGetAMResourceLimit()).
|
||||
isEqualTo(amResourceLimit);
|
||||
assertThat(queue.getUserAMResourceLimit()).isEqualTo(
|
||||
Resource.newInstance(80*GB, 1));
|
||||
Resource.newInstance(80*GB, 1));
|
||||
|
||||
// Assert in metrics
|
||||
assertThat(queue.getMetrics().getAMResourceLimitMB()).isEqualTo(
|
||||
@ -307,10 +308,8 @@ public void testLimitsComputation() throws Exception {
|
||||
assertThat(queue.getMetrics().getAMResourceLimitVCores()).isEqualTo(
|
||||
amResourceLimit.getVirtualCores());
|
||||
|
||||
assertEquals(
|
||||
(int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
|
||||
queue.getMetrics().getAvailableMB()
|
||||
);
|
||||
assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
|
||||
queue.getMetrics().getAvailableMB());
|
||||
|
||||
// Add some nodes to the cluster & test new limits
|
||||
clusterResource = Resources.createResource(120 * 16 * GB);
|
||||
@ -322,10 +321,8 @@ public void testLimitsComputation() throws Exception {
|
||||
assertThat(queue.getUserAMResourceLimit()).isEqualTo(
|
||||
Resource.newInstance(96*GB, 1));
|
||||
|
||||
assertEquals(
|
||||
(int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
|
||||
queue.getMetrics().getAvailableMB()
|
||||
);
|
||||
assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
|
||||
queue.getMetrics().getAvailableMB());
|
||||
|
||||
// should return -1 if per queue setting not set
|
||||
assertEquals(
|
||||
@ -343,11 +340,10 @@ public void testLimitsComputation() throws Exception {
|
||||
assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
|
||||
|
||||
// should default to global setting if per queue setting not set
|
||||
assertEquals(
|
||||
(long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
|
||||
assertEquals((long)
|
||||
CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
|
||||
(long)csConf.getMaximumApplicationMasterResourcePerQueuePercent(
|
||||
queue.getQueuePath())
|
||||
);
|
||||
queue.getQueuePath()));
|
||||
|
||||
// Change the per-queue max AM resources percentage.
|
||||
csConf.setFloat(PREFIX + queue.getQueuePath()
|
||||
@ -365,10 +361,9 @@ public void testLimitsComputation() throws Exception {
|
||||
|
||||
queue = (LeafQueue)queues.get(A);
|
||||
|
||||
assertEquals((long) 0.5,
|
||||
assertEquals((long) 0.5,
|
||||
(long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(
|
||||
queue.getQueuePath())
|
||||
);
|
||||
queue.getQueuePath()));
|
||||
|
||||
assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo(
|
||||
Resource.newInstance(800 * GB, 1));
|
||||
@ -579,8 +574,8 @@ public void testHeadroom() throws Exception {
|
||||
// Say cluster has 100 nodes of 16G each
|
||||
Resource clusterResource = Resources.createResource(100 * 16 * GB);
|
||||
|
||||
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB),
|
||||
Resources.createResource(16*GB), clusterResource);
|
||||
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
|
||||
Resources.createResource(GB), Resources.createResource(16*GB), clusterResource);
|
||||
CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager();
|
||||
CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
|
||||
|
||||
|
||||
@ -57,7 +57,6 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
|
||||
|
||||
@ -58,7 +58,6 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
|
||||
|
||||
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user