diff --git a/CHANGES.txt b/CHANGES.txt index c712e63d11..04d69c9792 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -556,6 +556,9 @@ Trunk (unreleased changes) HADOOP-5879. Read compression level and strategy from Configuration for gzip compression. (He Yongqiang via cdouglas) + HADOOP-6216. Support comments in host files. (Ravi Phulari and Dmytro + Molkov via szetszwo) + OPTIMIZATIONS HADOOP-5595. NameNode does not need to run a replicator to choose a diff --git a/src/java/org/apache/hadoop/util/HostsFileReader.java b/src/java/org/apache/hadoop/util/HostsFileReader.java index 89cdb39b0c..254845f8a0 100644 --- a/src/java/org/apache/hadoop/util/HostsFileReader.java +++ b/src/java/org/apache/hadoop/util/HostsFileReader.java @@ -58,7 +58,12 @@ private void readFileToSet(String filename, Set set) throws IOException String[] nodes = line.split("[ \t\n\f\r]+"); if (nodes != null) { for (int i = 0; i < nodes.length; i++) { + if (nodes[i].trim().startsWith("#")) { + // Everything from now on is a comment + break; + } if (!nodes[i].equals("")) { + LOG.info("Adding " + nodes[i] + " to the list of hosts from " + filename); set.add(nodes[i]); // might need to add canonical name } } diff --git a/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java b/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java new file mode 100644 index 0000000000..9fbd02cfd7 --- /dev/null +++ b/src/test/core/org/apache/hadoop/util/TestHostsFileReader.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.File; +import java.io.FileWriter; +import java.util.Set; + +import org.junit.*; +import static org.junit.Assert.*; + +/* + * Test for HostsFileReader.java + * + */ +public class TestHostsFileReader { + + // Using /test/build/data/tmp directory to store temprory files + final String HOSTS_TEST_DIR = new File(System.getProperty( + "test.build.data", "/tmp")).getAbsolutePath(); + File EXCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.exclude"); + File INCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.include"); + String excludesFile = HOSTS_TEST_DIR + "/dfs.exclude"; + String includesFile = HOSTS_TEST_DIR + "/dfs.include"; + private Set includes; + private Set excludes; + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + // Delete test files after running tests + EXCLUDES_FILE.delete(); + INCLUDES_FILE.delete(); + + } + + /* + * 1.Create dfs.exclude,dfs.include file + * 2.Write host names per line + * 3.Write comments starting with # + * 4.Close file + * 5.Compare if number of hosts reported by HostsFileReader + * are equal to the number of hosts written + */ + @Test + public void testHostsFileReader() throws Exception { + + FileWriter efw = new FileWriter(excludesFile); + FileWriter ifw = new FileWriter(includesFile); + + efw.write("#DFS-Hosts-excluded\n"); + efw.write("somehost1\n"); + efw.write("#This-is-comment\n"); + efw.write("somehost2\n"); + efw.write("somehost3 # host3\n"); + efw.write("somehost4\n"); + efw.write("somehost4 somehost5\n"); + efw.close(); + + ifw.write("#Hosts-in-DFS\n"); + ifw.write("somehost1\n"); + ifw.write("somehost2\n"); + ifw.write("somehost3\n"); + ifw.write("#This-is-comment\n"); + ifw.write("somehost4 # host4\n"); + ifw.write("somehost4 somehost5\n"); + ifw.close(); + + HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); + + int includesLen = hfp.getHosts().size(); + int excludesLen = hfp.getExcludedHosts().size(); + + assertEquals(5, includesLen); + assertEquals(5, excludesLen); + + assertTrue(hfp.getHosts().contains("somehost5")); + assertFalse(hfp.getHosts().contains("host3")); + + assertTrue(hfp.getExcludedHosts().contains("somehost5")); + assertFalse(hfp.getExcludedHosts().contains("host4")); + + } + + /* + * Test for null file + */ + @Test + public void testHostFileReaderWithNull() throws Exception { + FileWriter efw = new FileWriter(excludesFile); + FileWriter ifw = new FileWriter(includesFile); + + efw.close(); + + ifw.close(); + + HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); + + int includesLen = hfp.getHosts().size(); + int excludesLen = hfp.getExcludedHosts().size(); + + // TestCase1: Check if lines beginning with # are ignored + assertEquals(0, includesLen); + assertEquals(0, excludesLen); + + // TestCase2: Check if given host names are reported by getHosts and + // getExcludedHosts + assertFalse(hfp.getHosts().contains("somehost5")); + + assertFalse(hfp.getExcludedHosts().contains("somehost5")); + } + + /* + * Check if only comments can be written to hosts file + */ + @Test + public void testHostFileReaderWithCommentsOnly() throws Exception { + FileWriter efw = new FileWriter(excludesFile); + FileWriter ifw = new FileWriter(includesFile); + + efw.write("#DFS-Hosts-excluded\n"); + efw.close(); + + ifw.write("#Hosts-in-DFS\n"); + ifw.close(); + + HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); + + int includesLen = hfp.getHosts().size(); + int excludesLen = hfp.getExcludedHosts().size(); + + assertEquals(0, includesLen); + assertEquals(0, excludesLen); + + assertFalse(hfp.getHosts().contains("somehost5")); + + assertFalse(hfp.getExcludedHosts().contains("somehost5")); + + } + + /* + * Test if spaces are allowed in host names + */ + @Test + public void testHostFileReaderWithSpaces() throws Exception { + FileWriter efw = new FileWriter(excludesFile); + FileWriter ifw = new FileWriter(includesFile); + + efw.write("#DFS-Hosts-excluded\n"); + efw.write(" somehost somehost2"); + efw.write(" somehost3 # somehost4"); + efw.close(); + + ifw.write("#Hosts-in-DFS\n"); + ifw.write(" somehost somehost2"); + ifw.write(" somehost3 # somehost4"); + ifw.close(); + + HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); + + int includesLen = hfp.getHosts().size(); + int excludesLen = hfp.getExcludedHosts().size(); + + assertEquals(3, includesLen); + assertEquals(3, excludesLen); + + assertTrue(hfp.getHosts().contains("somehost3")); + assertFalse(hfp.getHosts().contains("somehost5")); + assertFalse(hfp.getHosts().contains("somehost4")); + + assertTrue(hfp.getExcludedHosts().contains("somehost3")); + assertFalse(hfp.getExcludedHosts().contains("somehost5")); + assertFalse(hfp.getExcludedHosts().contains("somehost4")); + + } + + /* + * Test if spaces , tabs and new lines are allowed + */ + @Test + public void testHostFileReaderWithTabs() throws Exception { + FileWriter efw = new FileWriter(excludesFile); + FileWriter ifw = new FileWriter(includesFile); + + efw.write("#DFS-Hosts-excluded\n"); + efw.write(" \n"); + efw.write(" somehost \t somehost2 \n somehost4"); + efw.write(" somehost3 \t # somehost5"); + efw.close(); + + ifw.write("#Hosts-in-DFS\n"); + ifw.write(" \n"); + ifw.write(" somehost \t somehost2 \n somehost4"); + ifw.write(" somehost3 \t # somehost5"); + ifw.close(); + + HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); + + int includesLen = hfp.getHosts().size(); + int excludesLen = hfp.getExcludedHosts().size(); + + assertEquals(4, includesLen); + assertEquals(4, excludesLen); + + assertTrue(hfp.getHosts().contains("somehost2")); + assertFalse(hfp.getHosts().contains("somehost5")); + + assertTrue(hfp.getExcludedHosts().contains("somehost2")); + assertFalse(hfp.getExcludedHosts().contains("somehost5")); + + } +}