From 6a61928fb790d553c942641761ea963b5e996ef6 Mon Sep 17 00:00:00 2001 From: Arun Suresh Date: Tue, 17 Nov 2015 15:50:34 -0800 Subject: [PATCH] YARN-4184. Remove update reservation state api from state store as its not used by ReservationSystem (Sean Po via asuresh) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../recovery/FileSystemRMStateStore.java | 12 ----- .../recovery/LeveldbRMStateStore.java | 8 --- .../recovery/MemoryRMStateStore.java | 17 ------- .../recovery/NullRMStateStore.java | 10 ---- .../recovery/RMStateStore.java | 51 ------------------- .../recovery/RMStateStoreEventType.java | 1 - .../recovery/ZKRMStateStore.java | 11 ---- .../recovery/RMStateStoreTestBase.java | 5 +- 9 files changed, 5 insertions(+), 113 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 98820743f9..f3b6b24cc1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -250,6 +250,9 @@ Release 2.8.0 - UNRELEASED YARN-1509. Make AMRMClient support send increase container request and get increased/decreased containers. (Meng Ding via wangda) + YARN-4184. Remove update reservation state api from state store as its not used by + ReservationSystem (Sean Po via asuresh) + IMPROVEMENTS YARN-644. Basic null check is not performed on passed in arguments before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index f02fe78949..a1cebf59e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -865,18 +865,6 @@ protected void storeReservationState( writeFileWithRetries(reservationPath, reservationData, true); } - @Override - protected void updateReservationState( - ReservationAllocationStateProto reservationAllocation, String planName, - String reservationIdName) throws Exception { - Path planCreatePath = getNodePath(reservationRoot, planName); - Path reservationPath = getNodePath(planCreatePath, reservationIdName); - LOG.info("Updating state for reservation " + reservationIdName + " from " + - "plan " + planName + " at path " + reservationPath); - byte[] reservationData = reservationAllocation.toByteArray(); - updateFile(reservationPath, reservationData, true); - } - @Override protected void removeReservationState( String planName, String reservationIdName) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index faaadb8018..afc6721352 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -623,14 +623,6 @@ protected void storeReservationState( } } - @Override - protected void updateReservationState( - ReservationAllocationStateProto reservationAllocation, String planName, - String reservationIdName) throws Exception { - storeReservationState(reservationAllocation, planName, - reservationIdName); - } - @Override protected void removeReservationState(String planName, String reservationIdName) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index b4f8805448..ce6addbe7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -243,23 +243,6 @@ protected synchronized void storeReservationState( planState.put(reservationId, reservationAllocation); } - @Override - protected synchronized void updateReservationState( - ReservationAllocationStateProto reservationAllocation, String planName, - String reservationIdName) throws Exception { - LOG.info("Updating reservationallocation for " + reservationIdName + " " + - "for plan " + planName); - Map planState = - state.getReservationState().get(planName); - if (planState == null) { - throw new YarnRuntimeException("State for plan " + planName + " does " + - "not exist"); - } - ReservationId reservationId = - ReservationId.parseReservationId(reservationIdName); - planState.put(reservationId, reservationAllocation); - } - @Override protected synchronized void removeReservationState( String planName, String reservationIdName) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index ffb61536bf..96f77f5f77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -115,13 +115,6 @@ protected void removeReservationState(String planName, // Do nothing } - @Override - protected void updateReservationState( - ReservationAllocationStateProto reservationAllocation, String planName, - String reservationIdName) throws Exception { - // Do nothing - } - @Override public void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception { // Do nothing @@ -175,7 +168,4 @@ public void deleteStore() throws Exception { public void removeApplication(ApplicationId removeAppId) throws Exception { // Do nothing } - - - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index a2cf517d87..9a2e02a7f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -162,10 +162,6 @@ RMStateStoreEventType.REMOVE_APP, new RemoveAppTransition()) EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), RMStateStoreEventType.STORE_RESERVATION, new StoreReservationAllocationTransition()) - .addTransition(RMStateStoreState.ACTIVE, - EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), - RMStateStoreEventType.UPDATE_RESERVATION, - new UpdateReservationAllocationTransition()) .addTransition(RMStateStoreState.ACTIVE, EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), RMStateStoreEventType.REMOVE_RESERVATION, @@ -187,7 +183,6 @@ RMStateStoreEventType.REMOVE_APP, new RemoveAppTransition()) RMStateStoreEventType.UPDATE_DELEGATION_TOKEN, RMStateStoreEventType.UPDATE_AMRM_TOKEN, RMStateStoreEventType.STORE_RESERVATION, - RMStateStoreEventType.UPDATE_RESERVATION, RMStateStoreEventType.REMOVE_RESERVATION)); private final StateMachine { - @Override - public RMStateStoreState transition(RMStateStore store, - RMStateStoreEvent event) { - if (!(event instanceof RMStateStoreStoreReservationEvent)) { - // should never happen - LOG.error("Illegal event type: " + event.getClass()); - return RMStateStoreState.ACTIVE; - } - boolean isFenced = false; - RMStateStoreStoreReservationEvent reservationEvent = - (RMStateStoreStoreReservationEvent) event; - try { - LOG.info("Updating reservation allocation." + reservationEvent - .getReservationIdName()); - store.updateReservationState( - reservationEvent.getReservationAllocation(), - reservationEvent.getPlanName(), - reservationEvent.getReservationIdName()); - } catch (Exception e) { - LOG.error("Error while updating reservation allocation.", e); - isFenced = store.notifyStoreOperationFailedInternal(e); - } - return finalState(isFenced); - } - } - private static class RemoveReservationAllocationTransition implements MultipleArcTransition { @@ -939,14 +905,6 @@ public void storeNewReservation( planName, reservationIdName)); } - public void updateReservation( - ReservationAllocationStateProto reservationAllocation, - String planName, String reservationIdName) { - handleStoreEvent(new RMStateStoreStoreReservationEvent( - reservationAllocation, RMStateStoreEventType.UPDATE_RESERVATION, - planName, reservationIdName)); - } - public void removeReservation(String planName, String reservationIdName) { handleStoreEvent(new RMStateStoreStoreReservationEvent( null, RMStateStoreEventType.REMOVE_RESERVATION, @@ -970,15 +928,6 @@ protected abstract void storeReservationState( protected abstract void removeReservationState(String planName, String reservationIdName) throws Exception; - /** - * Blocking API - * Derived classes must implement this method to update the state of - * a reservation allocation. - */ - protected abstract void updateReservationState( - ReservationAllocationStateProto reservationAllocation, String planName, - String reservationIdName) throws Exception; - /** * Blocking API * Derived classes must implement this method to remove the state of diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java index 6f8dca75d0..492826de98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java @@ -34,6 +34,5 @@ public enum RMStateStoreEventType { UPDATE_DELEGATION_TOKEN, UPDATE_AMRM_TOKEN, STORE_RESERVATION, - UPDATE_RESERVATION, REMOVE_RESERVATION, } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index a86b60c7f9..ca0f4ac03c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -845,17 +845,6 @@ protected synchronized void storeReservationState( trx.commit(); } - @Override - protected synchronized void updateReservationState( - ReservationAllocationStateProto reservationAllocation, String planName, - String reservationIdName) - throws Exception { - SafeTransaction trx = new SafeTransaction(); - addOrUpdateReservationState( - reservationAllocation, planName, reservationIdName, trx, true); - trx.commit(); - } - private void addOrUpdateReservationState( ReservationAllocationStateProto reservationAllocation, String planName, String reservationIdName, SafeTransaction trx, boolean isUpdate) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 7b05af3bcc..175f31fcb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -760,9 +760,8 @@ public void testReservationStateStore( minAlloc, hasGang); allocationStateProto = ReservationSystemUtil.buildStateProto(allocation); - rmContext.getStateStore().updateReservation( - allocationStateProto, - planName, reservationIdName); + rmContext.getStateStore().removeReservation(planName, reservationIdName); + rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName); // load state and verify updated reservation validateStoredReservation(