big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8656. Preserve compatibility of ClientProtocol#rollingUpgrade after finalization.

Browse Source
This commit is contained in:
Andrew Wang 2015-06-26 11:30:59 -07:00
parent de480d6c89
commit 60b858bfa6
6 changed files with 68 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -853,7 +853,8 @@ public boolean restoreFailedStorage(String arg)
/**
* Rolling upgrade operations.
* @param action either query, prepare or finalize.
* @return rolling upgrade information.
* @return rolling upgrade information. On query, if no upgrade is in
* progress, returns null.
*/
@Idempotent
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1110,6 +1110,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8626. Reserved RBW space is not released if creation of RBW File
fails. (kanaka kumar avvaru via Arpit Agarwal)
HDFS08656. Preserve compatibility of ClientProtocol#rollingUpgrade after
finalization. (wang)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -6757,10 +6757,12 @@ RollingUpgradeInfo queryRollingUpgrade() throws IOException {
checkOperation(OperationCategory.READ);
readLock();
try {
if (rollingUpgradeInfo != null) {
if (!isRollingUpgrade()) {
return null;
}
Preconditions.checkNotNull(rollingUpgradeInfo);
boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage();
rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage);
}
return rollingUpgradeInfo;
} finally {
readUnlock();

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
View File

@ -81,9 +81,10 @@ public interface NameNodeMXBean {
public boolean isUpgradeFinalized();
/**
* Gets the RollingUpgrade information
* Gets the RollingUpgrade information.
*
* @return Rolling upgrade information
* @return Rolling upgrade information if an upgrade is in progress. Else
* (e.g. if there is no upgrade or the upgrade is finalized), returns null.
*/
public RollingUpgradeInfo.Bean getRollingUpgradeStatus();

63
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
View File

@ -19,8 +19,18 @@
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.concurrent.ThreadLocalRandom;
import javax.management.AttributeNotFoundException;
import javax.management.InstanceNotFoundException;
import javax.management.MBeanException;
import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.management.ReflectionException;
import javax.management.openmbean.CompositeDataSupport;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@ -45,7 +55,9 @@
import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
/**
* This class tests rolling upgrade.
@ -56,7 +68,7 @@ public class TestRollingUpgrade {
public static void runCmd(DFSAdmin dfsadmin, boolean success,
String... args) throws Exception {
if (success) {
Assert.assertEquals(0, dfsadmin.run(args));
assertEquals(0, dfsadmin.run(args));
} else {
Assert.assertTrue(dfsadmin.run(args) != 0);
}
@ -86,6 +98,7 @@ public void testDFSAdminRollingUpgradeCommands() throws Exception {
//illegal argument "abc" to rollingUpgrade option
runCmd(dfsadmin, false, "-rollingUpgrade", "abc");
checkMxBeanIsNull();
//query rolling upgrade
runCmd(dfsadmin, true, "-rollingUpgrade");
@ -96,11 +109,16 @@ public void testDFSAdminRollingUpgradeCommands() throws Exception {
//query rolling upgrade
runCmd(dfsadmin, true, "-rollingUpgrade", "query");
checkMxBean();
dfs.mkdirs(bar);
//finalize rolling upgrade
runCmd(dfsadmin, true, "-rollingUpgrade", "finalize");
// RollingUpgradeInfo should be null after finalization, both via
// Java API and in JMX
assertNull(dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
checkMxBeanIsNull();
dfs.mkdirs(baz);
@ -197,7 +215,7 @@ public void testRollingUpgradeWithQJM() throws Exception {
LOG.info("START\n" + info1);
//query rolling upgrade
Assert.assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
dfs.mkdirs(bar);
cluster.shutdown();
@ -218,13 +236,13 @@ public void testRollingUpgradeWithQJM() throws Exception {
Assert.assertFalse(dfs2.exists(baz));
//query rolling upgrade in cluster2
Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
dfs2.mkdirs(baz);
LOG.info("RESTART cluster 2");
cluster2.restartNameNode();
Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
Assert.assertTrue(dfs2.exists(foo));
Assert.assertTrue(dfs2.exists(bar));
Assert.assertTrue(dfs2.exists(baz));
@ -238,7 +256,7 @@ public void testRollingUpgradeWithQJM() throws Exception {
LOG.info("RESTART cluster 2 again");
cluster2.restartNameNode();
Assert.assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
Assert.assertTrue(dfs2.exists(foo));
Assert.assertTrue(dfs2.exists(bar));
Assert.assertTrue(dfs2.exists(baz));
@ -259,8 +277,30 @@ public void testRollingUpgradeWithQJM() throws Exception {
}
}
private static CompositeDataSupport getBean()
throws MalformedObjectNameException, MBeanException,
AttributeNotFoundException, InstanceNotFoundException,
ReflectionException {
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
ObjectName mxbeanName =
new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo");
return (CompositeDataSupport)mbs.getAttribute(mxbeanName,
"RollingUpgradeStatus");
}
private static void checkMxBeanIsNull() throws Exception {
CompositeDataSupport ruBean = getBean();
assertNull(ruBean);
}
private static void checkMxBean() throws Exception {
CompositeDataSupport ruBean = getBean();
assertNotEquals(0l, ruBean.get("startTime"));
assertEquals(0l, ruBean.get("finalizeTime"));
}
@Test
public void testRollback() throws IOException {
public void testRollback() throws Exception {
// start a cluster
final Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
@ -279,10 +319,13 @@ public void testRollback() throws IOException {
out.write(data, 0, data.length);
out.close();
checkMxBeanIsNull();
startRollingUpgrade(foo, bar, file, data, cluster);
checkMxBean();
cluster.getFileSystem().rollEdits();
cluster.getFileSystem().rollEdits();
rollbackRollingUpgrade(foo, bar, file, data, cluster);
checkMxBeanIsNull();
startRollingUpgrade(foo, bar, file, data, cluster);
cluster.getFileSystem().rollEdits();
@ -356,18 +399,18 @@ public void testDFSAdminDatanodeUpgradeControlCommands() throws Exception {
// check the datanode
final String dnAddr = dn.getDatanodeId().getIpcAddr(false);
final String[] args1 = {"-getDatanodeInfo", dnAddr};
Assert.assertEquals(0, dfsadmin.run(args1));
runCmd(dfsadmin, true, args1);
// issue shutdown to the datanode.
final String[] args2 = {"-shutdownDatanode", dnAddr, "upgrade" };
Assert.assertEquals(0, dfsadmin.run(args2));
runCmd(dfsadmin, true, args2);
// the datanode should be down.
Thread.sleep(2000);
Assert.assertFalse("DataNode should exit", dn.isDatanodeUp());
// ping should fail.
Assert.assertEquals(-1, dfsadmin.run(args1));
assertEquals(-1, dfsadmin.run(args1));
} finally {
if (cluster != null) cluster.shutdown();
}

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
View File

@ -28,7 +28,6 @@
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.util.VersionInfo;
@ -46,8 +45,6 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op;
import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@ -197,6 +194,8 @@ public void testNameNodeMXBeanInfo() throws Exception {
assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
cluster.getDataNodes().size(),
mbs.getAttribute(mxbeanName, "CacheCapacity"));
assertNull("RollingUpgradeInfo should be null when there is no rolling"
+ " upgrade", mbs.getAttribute(mxbeanName, "RollingUpgradeStatus"));
} finally {
if (cluster != null) {
for (URI dir : cluster.getNameDirs(0)) {