big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-6216. Support comments in host files. Contributed by Ravi Phulari and Dmytro Molkov

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@815524 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2009-09-15 22:18:21 +00:00
parent 78cff65f33
commit 9164c6319e
3 changed files with 238 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -556,6 +556,9 @@ Trunk (unreleased changes)
HADOOP-5879. Read compression level and strategy from Configuration for
gzip compression. (He Yongqiang via cdouglas)
HADOOP-6216. Support comments in host files. (Ravi Phulari and Dmytro
Molkov via szetszwo)
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a

5
src/java/org/apache/hadoop/util/HostsFileReader.java
View File

@ -58,7 +58,12 @@ private void readFileToSet(String filename, Set<String> set) throws IOException
String[] nodes = line.split("[ \t\n\f\r]+");
if (nodes != null) {
for (int i = 0; i < nodes.length; i++) {
if (nodes[i].trim().startsWith("#")) {
// Everything from now on is a comment
break;
}
if (!nodes[i].equals("")) {
LOG.info("Adding " + nodes[i] + " to the list of hosts from " + filename);
set.add(nodes[i]); // might need to add canonical name
}
}

230
src/test/core/org/apache/hadoop/util/TestHostsFileReader.java Normal file
View File

@ -0,0 +1,230 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import java.io.File;
import java.io.FileWriter;
import java.util.Set;
import org.junit.*;
import static org.junit.Assert.*;
/*
* Test for HostsFileReader.java
*
*/
public class TestHostsFileReader {
// Using /test/build/data/tmp directory to store temprory files
final String HOSTS_TEST_DIR = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File EXCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.exclude");
File INCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.include");
String excludesFile = HOSTS_TEST_DIR + "/dfs.exclude";
String includesFile = HOSTS_TEST_DIR + "/dfs.include";
private Set<String> includes;
private Set<String> excludes;
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
// Delete test files after running tests
EXCLUDES_FILE.delete();
INCLUDES_FILE.delete();
}
/*
* 1.Create dfs.exclude,dfs.include file
* 2.Write host names per line
* 3.Write comments starting with #
* 4.Close file
* 5.Compare if number of hosts reported by HostsFileReader
* are equal to the number of hosts written
*/
@Test
public void testHostsFileReader() throws Exception {
FileWriter efw = new FileWriter(excludesFile);
FileWriter ifw = new FileWriter(includesFile);
efw.write("#DFS-Hosts-excluded\n");
efw.write("somehost1\n");
efw.write("#This-is-comment\n");
efw.write("somehost2\n");
efw.write("somehost3 # host3\n");
efw.write("somehost4\n");
efw.write("somehost4 somehost5\n");
efw.close();
ifw.write("#Hosts-in-DFS\n");
ifw.write("somehost1\n");
ifw.write("somehost2\n");
ifw.write("somehost3\n");
ifw.write("#This-is-comment\n");
ifw.write("somehost4 # host4\n");
ifw.write("somehost4 somehost5\n");
ifw.close();
HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
int includesLen = hfp.getHosts().size();
int excludesLen = hfp.getExcludedHosts().size();
assertEquals(5, includesLen);
assertEquals(5, excludesLen);
assertTrue(hfp.getHosts().contains("somehost5"));
assertFalse(hfp.getHosts().contains("host3"));
assertTrue(hfp.getExcludedHosts().contains("somehost5"));
assertFalse(hfp.getExcludedHosts().contains("host4"));
}
/*
* Test for null file
*/
@Test
public void testHostFileReaderWithNull() throws Exception {
FileWriter efw = new FileWriter(excludesFile);
FileWriter ifw = new FileWriter(includesFile);
efw.close();
ifw.close();
HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
int includesLen = hfp.getHosts().size();
int excludesLen = hfp.getExcludedHosts().size();
// TestCase1: Check if lines beginning with # are ignored
assertEquals(0, includesLen);
assertEquals(0, excludesLen);
// TestCase2: Check if given host names are reported by getHosts and
// getExcludedHosts
assertFalse(hfp.getHosts().contains("somehost5"));
assertFalse(hfp.getExcludedHosts().contains("somehost5"));
}
/*
* Check if only comments can be written to hosts file
*/
@Test
public void testHostFileReaderWithCommentsOnly() throws Exception {
FileWriter efw = new FileWriter(excludesFile);
FileWriter ifw = new FileWriter(includesFile);
efw.write("#DFS-Hosts-excluded\n");
efw.close();
ifw.write("#Hosts-in-DFS\n");
ifw.close();
HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
int includesLen = hfp.getHosts().size();
int excludesLen = hfp.getExcludedHosts().size();
assertEquals(0, includesLen);
assertEquals(0, excludesLen);
assertFalse(hfp.getHosts().contains("somehost5"));
assertFalse(hfp.getExcludedHosts().contains("somehost5"));
}
/*
* Test if spaces are allowed in host names
*/
@Test
public void testHostFileReaderWithSpaces() throws Exception {
FileWriter efw = new FileWriter(excludesFile);
FileWriter ifw = new FileWriter(includesFile);
efw.write("#DFS-Hosts-excluded\n");
efw.write(" somehost somehost2");
efw.write(" somehost3 # somehost4");
efw.close();
ifw.write("#Hosts-in-DFS\n");
ifw.write(" somehost somehost2");
ifw.write(" somehost3 # somehost4");
ifw.close();
HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
int includesLen = hfp.getHosts().size();
int excludesLen = hfp.getExcludedHosts().size();
assertEquals(3, includesLen);
assertEquals(3, excludesLen);
assertTrue(hfp.getHosts().contains("somehost3"));
assertFalse(hfp.getHosts().contains("somehost5"));
assertFalse(hfp.getHosts().contains("somehost4"));
assertTrue(hfp.getExcludedHosts().contains("somehost3"));
assertFalse(hfp.getExcludedHosts().contains("somehost5"));
assertFalse(hfp.getExcludedHosts().contains("somehost4"));
}
/*
* Test if spaces , tabs and new lines are allowed
*/
@Test
public void testHostFileReaderWithTabs() throws Exception {
FileWriter efw = new FileWriter(excludesFile);
FileWriter ifw = new FileWriter(includesFile);
efw.write("#DFS-Hosts-excluded\n");
efw.write(" \n");
efw.write(" somehost \t somehost2 \n somehost4");
efw.write(" somehost3 \t # somehost5");
efw.close();
ifw.write("#Hosts-in-DFS\n");
ifw.write(" \n");
ifw.write(" somehost \t somehost2 \n somehost4");
ifw.write(" somehost3 \t # somehost5");
ifw.close();
HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile);
int includesLen = hfp.getHosts().size();
int excludesLen = hfp.getExcludedHosts().size();
assertEquals(4, includesLen);
assertEquals(4, excludesLen);
assertTrue(hfp.getHosts().contains("somehost2"));
assertFalse(hfp.getHosts().contains("somehost5"));
assertTrue(hfp.getExcludedHosts().contains("somehost2"));
assertFalse(hfp.getExcludedHosts().contains("somehost5"));
}
}