big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-4974. Add Idempotent and AtMostOnce annotations to namenode protocol methods. Contributed by Suresh Srinivas.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1505911 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Suresh Srinivas 2013-07-23 06:40:33 +00:00
parent 9dbd2e7efc
commit d35f66f3f0
10 changed files with 145 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.KerberosInfo;
@ -106,6 +107,7 @@ public RequestSource getSource() {
* @throws IOException
* if other errors happen
*/
@Idempotent
public void monitorHealth() throws HealthCheckFailedException,
AccessControlException,
IOException;
@ -121,6 +123,7 @@ public void monitorHealth() throws HealthCheckFailedException,
* @throws IOException
* if other errors happen
*/
@Idempotent
public void transitionToActive(StateChangeRequestInfo reqInfo)
throws ServiceFailedException,
AccessControlException,
@ -137,6 +140,7 @@ public void transitionToActive(StateChangeRequestInfo reqInfo)
* @throws IOException
* if other errors happen
*/
@Idempotent
public void transitionToStandby(StateChangeRequestInfo reqInfo)
throws ServiceFailedException,
AccessControlException,
@ -153,6 +157,7 @@ public void transitionToStandby(StateChangeRequestInfo reqInfo)
* if other errors happen
* @see HAServiceStatus
*/
@Idempotent
public HAServiceStatus getServiceStatus() throws AccessControlException,
IOException;
}

41
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AtMostOnce.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.retry;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.apache.hadoop.classification.InterfaceStability;
/**
* Used to mark certain methods of an interface with at-most-once semantics.
*
* Server must guarantee that methods are executed at most once, by keeping
* a retry cache. The previous response must be returned when duplicate
* requests are received. Because of these guarantee, a client can retry
* this request on failover and other network failure conditions.
*/
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
@InterfaceStability.Evolving
public @interface AtMostOnce {
}

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RefreshUserMappingsProtocol.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.KerberosInfo;
/**
@ -43,12 +44,13 @@ public interface RefreshUserMappingsProtocol {
* Refresh user to group mappings.
* @throws IOException
*/
@Idempotent
public void refreshUserToGroupsMappings() throws IOException;
/**
* Refresh superuser proxy group list
* @throws IOException
*/
public void refreshSuperUserGroupsConfiguration()
throws IOException;
@Idempotent
public void refreshSuperUserGroupsConfiguration() throws IOException;
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/RefreshAuthorizationPolicyProtocol.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.KerberosInfo;
/**
@ -42,5 +43,6 @@ public interface RefreshAuthorizationPolicyProtocol {
* Refresh the service-level authorization policy in-effect.
* @throws IOException
*/
@Idempotent
void refreshServiceAcl() throws IOException;
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetUserMappingsProtocol.java
View File

@ -21,6 +21,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.retry.Idempotent;
/**
* Protocol implemented by the Name Node and Job Tracker which maps users to
@ -41,5 +42,6 @@ public interface GetUserMappingsProtocol {
* @return The set of groups the user belongs to.
* @throws IOException
*/
@Idempotent
public String[] getGroupsForUser(String user) throws IOException;
}

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -335,6 +335,9 @@ Release 2.1.0-beta - 2013-07-02
HDFS-4373. Add HTTP API for querying NameNode startup progress. (cnauroth)
HDFS-4374. Display NameNode startup progress in UI. (cnauroth)
HDFS-4974. Add Idempotent and AtMostOnce annotations to namenode
protocol methods. (suresh)
IMPROVEMENTS

23
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.KerberosInfo;
@ -139,7 +140,7 @@ public LocatedBlocks getBlockLocations(String src,
* <p>
* Blocks have a maximum size. Clients that intend to create
* multi-block files must also use
* {@link #addBlock(String, String, ExtendedBlock, DatanodeInfo[])}
* {@link #addBlock}
*
* @param src path of the file being created.
* @param masked masked permission.
@ -170,7 +171,10 @@ public LocatedBlocks getBlockLocations(String src,
*
* RuntimeExceptions:
* @throws InvalidPathException Path <code>src</code> is invalid
* <p>
* <em>Note that create with {@link CreateFlag#OVERWRITE} is idempotent.</em>
*/
@AtMostOnce
public HdfsFileStatus create(String src, FsPermission masked,
String clientName, EnumSetWritable<CreateFlag> flag,
boolean createParent, short replication, long blockSize)
@ -204,6 +208,7 @@ public HdfsFileStatus create(String src, FsPermission masked,
* RuntimeExceptions:
* @throws UnsupportedOperationException if append is not supported
*/
@AtMostOnce
public LocatedBlock append(String src, String clientName)
throws AccessControlException, DSQuotaExceededException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
@ -409,6 +414,7 @@ public boolean complete(String src, String clientName,
* @throws SnapshotAccessControlException if path is in RO snapshot
* @throws IOException an I/O error occurred
*/
@AtMostOnce
public boolean rename(String src, String dst)
throws UnresolvedLinkException, SnapshotAccessControlException, IOException;
@ -422,6 +428,7 @@ public boolean rename(String src, String dst)
* contains a symlink
* @throws SnapshotAccessControlException if path is in RO snapshot
*/
@AtMostOnce
public void concat(String trg, String[] srcs)
throws IOException, UnresolvedLinkException, SnapshotAccessControlException;
@ -460,6 +467,7 @@ public void concat(String trg, String[] srcs)
* @throws SnapshotAccessControlException if path is in RO snapshot
* @throws IOException If an I/O error occurred
*/
@AtMostOnce
public void rename2(String src, String dst, Options.Rename... options)
throws AccessControlException, DSQuotaExceededException,
FileAlreadyExistsException, FileNotFoundException,
@ -484,6 +492,7 @@ public void rename2(String src, String dst, Options.Rename... options)
* @throws SnapshotAccessControlException if path is in RO snapshot
* @throws IOException If an I/O error occurred
*/
@AtMostOnce
public boolean delete(String src, boolean recursive)
throws AccessControlException, FileNotFoundException, SafeModeException,
UnresolvedLinkException, SnapshotAccessControlException, IOException;
@ -704,6 +713,7 @@ public boolean setSafeMode(HdfsConstants.SafeModeAction action, boolean isChecke
* @throws AccessControlException if the superuser privilege is violated.
* @throws IOException if image creation failed.
*/
@AtMostOnce
public void saveNamespace() throws AccessControlException, IOException;
@ -725,6 +735,7 @@ public boolean setSafeMode(HdfsConstants.SafeModeAction action, boolean isChecke
*
* @throws AccessControlException if the superuser privilege is violated.
*/
@Idempotent
public boolean restoreFailedStorage(String arg)
throws AccessControlException, IOException;
@ -732,6 +743,7 @@ public boolean restoreFailedStorage(String arg)
* Tells the namenode to reread the hosts and exclude files.
* @throws IOException
*/
@Idempotent
public void refreshNodes() throws IOException;
/**
@ -741,6 +753,7 @@ public boolean restoreFailedStorage(String arg)
*
* @throws IOException
*/
@Idempotent
public void finalizeUpgrade() throws IOException;
/**
@ -763,6 +776,7 @@ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
*
* @throws IOException
*/
@Idempotent
public void metaSave(String filename) throws IOException;
/**
@ -918,6 +932,7 @@ public void setTimes(String src, long mtime, long atime)
* @throws SnapshotAccessControlException if path is in RO snapshot
* @throws IOException If an I/O error occurred
*/
@AtMostOnce
public void createSymlink(String target, String link, FsPermission dirPerm,
boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
@ -965,6 +980,7 @@ public LocatedBlock updateBlockForPipeline(ExtendedBlock block,
* @param newNodes datanodes in the pipeline
* @throws IOException if any error occurs
*/
@AtMostOnce
public void updatePipeline(String clientName, ExtendedBlock oldBlock,
ExtendedBlock newBlock, DatanodeID[] newNodes)
throws IOException;
@ -997,6 +1013,7 @@ public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
* @param token delegation token
* @throws IOException
*/
@Idempotent
public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
throws IOException;
@ -1005,6 +1022,7 @@ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
* DataTransferProtocol to/from DataNodes.
* @throws IOException
*/
@Idempotent
public DataEncryptionKey getDataEncryptionKey() throws IOException;
/**
@ -1014,6 +1032,7 @@ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
* @return the snapshot path.
* @throws IOException
*/
@AtMostOnce
public String createSnapshot(String snapshotRoot, String snapshotName)
throws IOException;
@ -1023,6 +1042,7 @@ public String createSnapshot(String snapshotRoot, String snapshotName)
* @param snapshotName Name of the snapshot for the snapshottable directory
* @throws IOException
*/
@AtMostOnce
public void deleteSnapshot(String snapshotRoot, String snapshotName)
throws IOException;
@ -1033,6 +1053,7 @@ public void deleteSnapshot(String snapshotRoot, String snapshotName)
* @param snapshotNewName new name of the snapshot
* @throws IOException
*/
@AtMostOnce
public void renameSnapshot(String snapshotRoot, String snapshotOldName,
String snapshotNewName) throws IOException;

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
View File

@ -25,6 +25,8 @@
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.KerberosInfo;
/**********************************************************************
@ -81,6 +83,7 @@ public interface DatanodeProtocol {
* @return the given {@link org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration} with
* updated registration information
*/
@Idempotent
public DatanodeRegistration registerDatanode(DatanodeRegistration registration
) throws IOException;
@ -98,6 +101,7 @@ public DatanodeRegistration registerDatanode(DatanodeRegistration registration
* @param failedVolumes number of failed volumes
* @throws IOException on error
*/
@Idempotent
public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration,
StorageReport[] reports,
int xmitsInProgress,
@ -120,6 +124,7 @@ public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration,
* @return - the next command for DN to process.
* @throws IOException
*/
@Idempotent
public DatanodeCommand blockReport(DatanodeRegistration registration,
String poolId, StorageBlockReport[] reports) throws IOException;
@ -133,6 +138,7 @@ public DatanodeCommand blockReport(DatanodeRegistration registration,
* writes a new Block here, or another DataNode copies a Block to
* this DataNode, it will call blockReceived().
*/
@AtMostOnce
public void blockReceivedAndDeleted(DatanodeRegistration registration,
String poolId,
StorageReceivedDeletedBlocks[] rcvdAndDeletedBlocks)
@ -142,21 +148,25 @@ public void blockReceivedAndDeleted(DatanodeRegistration registration,
* errorReport() tells the NameNode about something that has gone
* awry. Useful for debugging.
*/
@Idempotent
public void errorReport(DatanodeRegistration registration,
int errorCode,
String msg) throws IOException;
@Idempotent
public NamespaceInfo versionRequest() throws IOException;
/**
* same as {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#reportBadBlocks(LocatedBlock[])}
* }
*/
@Idempotent
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException;
/**
* Commit block synchronization in lease recovery
*/
@AtMostOnce
public void commitBlockSynchronization(ExtendedBlock block,
long newgenerationstamp, long newlength,
boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
View File

@ -25,6 +25,8 @@
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.KerberosInfo;
/*****************************************************************************
@ -73,6 +75,7 @@ public interface NamenodeProtocol {
* @throws IOException if size is less than or equal to 0 or
datanode does not exist
*/
@Idempotent
public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
throws IOException;
@ -82,6 +85,7 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
* @return ExportedBlockKeys containing current block keys
* @throws IOException
*/
@Idempotent
public ExportedBlockKeys getBlockKeys() throws IOException;
/**
@ -90,11 +94,13 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
* case of a non-active node.
* @throws IOException
*/
@Idempotent
public long getTransactionID() throws IOException;
/**
* Get the transaction ID of the most recent checkpoint.
*/
@Idempotent
public long getMostRecentCheckpointTxId() throws IOException;
/**
@ -103,6 +109,7 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
* @throws IOException
* @return a unique token to identify this transaction.
*/
@Idempotent
public CheckpointSignature rollEditLog() throws IOException;
/**
@ -112,6 +119,7 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
* of the name-node
* @throws IOException
*/
@Idempotent
public NamespaceInfo versionRequest() throws IOException;
/**
@ -124,6 +132,7 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
* @param msg free text description of the error
* @throws IOException
*/
@Idempotent
public void errorReport(NamenodeRegistration registration,
int errorCode,
String msg) throws IOException;
@ -134,6 +143,7 @@ public void errorReport(NamenodeRegistration registration,
* @return {@link NamenodeRegistration} of the node,
* which this node has just registered with.
*/
@Idempotent
public NamenodeRegistration registerSubordinateNamenode(
NamenodeRegistration registration) throws IOException;
@ -151,6 +161,7 @@ public NamenodeRegistration registerSubordinateNamenode(
* @return {@link CheckpointCommand} if checkpoint is allowed.
* @throws IOException
*/
@AtMostOnce
public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
throws IOException;
@ -162,6 +173,7 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
* @param sig {@code CheckpointSignature} which identifies the checkpoint.
* @throws IOException
*/
@AtMostOnce
public void endCheckpoint(NamenodeRegistration registration,
CheckpointSignature sig) throws IOException;
@ -171,6 +183,7 @@ public void endCheckpoint(NamenodeRegistration registration,
* available to be fetched from the NameNode.
* @param sinceTxId return only logs that contain transactions >= sinceTxId
*/
@Idempotent
public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
throws IOException;
}

43
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestAnnotations.java Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.protocol;
import java.lang.reflect.Method;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.junit.Assert;
import org.junit.Test;
/**
* Tests to make sure all the protocol class public methods have
* either {@link Idempotent} or {@link AtMostOnce} once annotations.
*/
public class TestAnnotations {
@Test
public void checkAnnotations() {
Method[] methods = NamenodeProtocols.class.getMethods();
for (Method m : methods) {
Assert.assertTrue(
"Idempotent or AtMostOnce annotation is not present " + m,
m.isAnnotationPresent(Idempotent.class)
|| m.isAnnotationPresent(AtMostOnce.class));
}
}
}