big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-9010. Replace NameNode.DEFAULT_PORT with HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT config key. Contributed by Mingliang Liu.

Browse Source
This commit is contained in:
Haohui Mai 2015-09-14 18:22:52 -07:00
parent 7b5cf5352e
commit 76957a485b
11 changed files with 43 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -909,6 +909,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8996. Consolidate validateLog and scanLog in FJM#EditLogFile (Zhe
Zhang via Colin P. McCabe)
HDFS-9010. Replace NameNode.DEFAULT_PORT with HdfsClientConfigKeys.
DFS_NAMENODE_RPC_PORT_DEFAULT config key. (Mingliang Liu via wheat9)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
View File

@ -41,6 +41,7 @@
import org.apache.hadoop.hdfs.DFSInputStream;
import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
@ -49,7 +50,6 @@
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@ -77,7 +77,8 @@ public class Hdfs extends AbstractFileSystem {
* @throws IOException
*/
Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException {
super(theUri, HdfsConstants.HDFS_URI_SCHEME, true, NameNode.DEFAULT_PORT);
super(theUri, HdfsConstants.HDFS_URI_SCHEME, true,
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
if (!theUri.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) {
throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs");
@ -92,7 +93,7 @@ public class Hdfs extends AbstractFileSystem {
@Override
public int getUriDefaultPort() {
return NameNode.DEFAULT_PORT;
return HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
}
@Override

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -63,6 +63,7 @@
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@ -84,7 +85,6 @@
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@ -1497,7 +1497,7 @@ public Void next(final FileSystem fs, final Path p)
@Override
protected int getDefaultPort() {
return NameNode.DEFAULT_PORT;
return HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
}
@Override

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
View File

@ -509,7 +509,8 @@ public static <T> AbstractNNFailoverProxyProvider<T> createFailoverProxyProvider
// Check the port in the URI, if it is logical.
if (checkPort && providerNN.useLogicalURI()) {
int port = nameNodeUri.getPort();
if (port > 0 && port != NameNode.DEFAULT_PORT) {
if (port > 0 &&
port != HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) {
// Throwing here without any cleanup is fine since we have not
// actually created the underlying proxies yet.
throw new IOException("Port " + port + " specified in URI "

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
View File

@ -39,9 +39,9 @@
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@ -111,6 +111,7 @@
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY;
@ -314,7 +315,12 @@ public long getProtocolVersion(String protocol,
}
}
public static final int DEFAULT_PORT = 8020;
/**
* @deprecated Use {@link HdfsClientConfigKeys#DFS_NAMENODE_RPC_PORT_DEFAULT}
* instead.
*/
@Deprecated
public static final int DEFAULT_PORT = DFS_NAMENODE_RPC_PORT_DEFAULT;
public static final Logger LOG =
LoggerFactory.getLogger(NameNode.class.getName());
public static final Logger stateChangeLog =
@ -452,7 +458,7 @@ public String getClientNamenodeAddress() {
}
public static InetSocketAddress getAddress(String address) {
return NetUtils.createSocketAddr(address, DEFAULT_PORT);
return NetUtils.createSocketAddr(address, DFS_NAMENODE_RPC_PORT_DEFAULT);
}
/**
@ -509,7 +515,8 @@ public static InetSocketAddress getAddress(URI filesystemURI) {
public static URI getUri(InetSocketAddress namenode) {
int port = namenode.getPort();
String portString = port == DEFAULT_PORT ? "" : (":"+port);
String portString = (port == DFS_NAMENODE_RPC_PORT_DEFAULT) ?
"" : (":" + port);
return URI.create(HdfsConstants.HDFS_URI_SCHEME + "://"
+ namenode.getHostName()+portString);
}

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetUtils;
@ -77,7 +78,7 @@ public NNHAServiceTarget(Configuration conf,
"Unable to determine service address for namenode '" + nnId + "'");
}
this.addr = NetUtils.createSocketAddr(serviceAddr,
NameNode.DEFAULT_PORT);
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
this.autoFailoverEnabled = targetConf.getBoolean(
DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY,

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
View File

@ -41,6 +41,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.TestFileTruncate;
@ -89,7 +90,7 @@ public static void startUp() throws IOException {
cluster = new MiniDFSCluster.Builder(conf)
.format(true)
.numDataNodes(DATANODE_NUM)
.nameNodePort(NameNode.DEFAULT_PORT)
.nameNodePort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)
.waitSafeMode(true)
.build();
dfs = cluster.getFileSystem();

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
View File

@ -115,7 +115,8 @@ public void testDfsClientFailover() throws IOException, URISyntaxException {
// to include a port number.
Path withPort = new Path("hdfs://" +
HATestUtil.getLogicalHostname(cluster) + ":" +
NameNode.DEFAULT_PORT + "/" + TEST_FILE.toUri().getPath());
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + "/" +
TEST_FILE.toUri().getPath());
FileSystem fs2 = withPort.getFileSystem(fs.getConf());
assertTrue(fs2.exists(withPort));

16
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java
View File

@ -24,7 +24,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.junit.Test;
/** Test NameNode port defaulting code. */
@ -33,9 +35,9 @@ public class TestDefaultNameNodePort {
@Test
public void testGetAddressFromString() throws Exception {
assertEquals(NameNode.getAddress("foo").getPort(),
NameNode.DEFAULT_PORT);
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
assertEquals(NameNode.getAddress("hdfs://foo/").getPort(),
NameNode.DEFAULT_PORT);
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
assertEquals(NameNode.getAddress("hdfs://foo:555").getPort(),
555);
assertEquals(NameNode.getAddress("foo:555").getPort(),
@ -46,11 +48,13 @@ public void testGetAddressFromString() throws Exception {
public void testGetAddressFromConf() throws Exception {
Configuration conf = new HdfsConfiguration();
FileSystem.setDefaultUri(conf, "hdfs://foo/");
assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT);
assertEquals(NameNode.getAddress(conf).getPort(),
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
FileSystem.setDefaultUri(conf, "hdfs://foo:555/");
assertEquals(NameNode.getAddress(conf).getPort(), 555);
FileSystem.setDefaultUri(conf, "foo");
assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT);
assertEquals(NameNode.getAddress(conf).getPort(),
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
}
@Test
@ -58,7 +62,7 @@ public void testGetUri() {
assertEquals(NameNode.getUri(new InetSocketAddress("foo", 555)),
URI.create("hdfs://foo:555"));
assertEquals(NameNode.getUri(new InetSocketAddress("foo",
NameNode.DEFAULT_PORT)),
URI.create("hdfs://foo"));
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)),
URI.create("hdfs://foo"));
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
View File

@ -31,8 +31,8 @@
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.junit.Test;
@ -67,7 +67,7 @@ public void testBalancerWithHANameNodes() throws Exception {
assertEquals(capacities.length, racks.length);
int numOfDatanodes = capacities.length;
NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
nn1Conf.setIpcPort(NameNode.DEFAULT_PORT);
nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
Configuration copiedConf = new Configuration(conf);
cluster = new MiniDFSCluster.Builder(copiedConf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
View File

@ -50,6 +50,7 @@
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@ -99,7 +100,7 @@ public void setUp() throws IOException {
cluster = new MiniDFSCluster.Builder(conf)
.format(true)
.numDataNodes(DATANODE_NUM)
.nameNodePort(NameNode.DEFAULT_PORT)
.nameNodePort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)
.waitSafeMode(true)
.build();
fs = cluster.getFileSystem();
@ -1224,7 +1225,7 @@ static void restartCluster(StartupOption o)
NameNode.doRollback(conf, false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM)
.format(false)
.nameNodePort(NameNode.DEFAULT_PORT)
.nameNodePort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)
.startupOption(o==StartupOption.ROLLBACK ? StartupOption.REGULAR : o)
.dnStartupOption(o!=StartupOption.ROLLBACK ? StartupOption.REGULAR : o)
.build();