diff --git a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java index 71ed762b5a..b6310fd91b 100644 --- a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java +++ b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java @@ -27,8 +27,6 @@ import java.util.List; import java.util.StringTokenizer; -import junit.framework.TestCase; - import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; @@ -38,111 +36,117 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.log4j.Level; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; /** * test {@link HadoopArchives} */ -public class TestHadoopArchives extends TestCase { +public class TestHadoopArchives { - public static final String HADOOP_ARCHIVES_JAR = JarFinder.getJar(HadoopArchives.class); + public static final String HADOOP_ARCHIVES_JAR = JarFinder + .getJar(HadoopArchives.class); { - ((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class) - ).getLogger().setLevel(Level.ERROR); - ((Log4JLogger)org.apache.hadoop.ipc.Server.LOG - ).getLogger().setLevel(Level.ERROR); - ((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG - ).getLogger().setLevel(Level.ERROR); + ((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class)) + .getLogger().setLevel(Level.ERROR); + } private static final String inputDir = "input"; private Path inputPath; private MiniDFSCluster dfscluster; - private MiniMRCluster mapred; + + private Configuration conf; private FileSystem fs; private Path archivePath; - - static private Path createFile(Path dir, String filename, FileSystem fs - ) throws IOException { + + static private Path createFile(Path dir, String filename, FileSystem fs) + throws IOException { final Path f = new Path(dir, filename); - final FSDataOutputStream out = fs.create(f); + final FSDataOutputStream out = fs.create(f); out.write(filename.getBytes()); out.close(); return f; } - - protected void setUp() throws Exception { - super.setUp(); - dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null); + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + conf.set(CapacitySchedulerConfiguration.PREFIX + + CapacitySchedulerConfiguration.ROOT + "." + + CapacitySchedulerConfiguration.QUEUES, "default"); + conf.set(CapacitySchedulerConfiguration.PREFIX + + CapacitySchedulerConfiguration.ROOT + ".default." + + CapacitySchedulerConfiguration.CAPACITY, "100"); + dfscluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true) + .build(); + fs = dfscluster.getFileSystem(); - mapred = new MiniMRCluster(2, fs.getUri().toString(), 1); - inputPath = new Path(fs.getHomeDirectory(), inputDir); + inputPath = new Path(fs.getHomeDirectory(), inputDir); archivePath = new Path(fs.getHomeDirectory(), "archive"); fs.mkdirs(inputPath); createFile(inputPath, "a", fs); createFile(inputPath, "b", fs); createFile(inputPath, "c", fs); } - - protected void tearDown() throws Exception { + + @After + public void tearDown() throws Exception { try { - if (mapred != null) { - mapred.shutdown(); + if (dfscluster != null) { + dfscluster.shutdown(); } if (dfscluster != null) { dfscluster.shutdown(); } - } catch(Exception e) { + } catch (Exception e) { System.err.println(e); } - super.tearDown(); } - - + + @Test public void testRelativePath() throws Exception { fs.delete(archivePath, true); final Path sub1 = new Path(inputPath, "dir1"); fs.mkdirs(sub1); createFile(sub1, "a", fs); - final Configuration conf = mapred.createJobConf(); final FsShell shell = new FsShell(conf); final List originalPaths = lsr(shell, "input"); System.out.println("originalPath: " + originalPaths); final URI uri = fs.getUri(); - final String prefix = "har://hdfs-" + uri.getHost() +":" + uri.getPort() + final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort() + archivePath.toUri().getPath() + Path.SEPARATOR; { final String harName = "foo.har"; - final String[] args = { - "-archiveName", - harName, - "-p", - "input", - "*", - "archive" - }; - System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH, HADOOP_ARCHIVES_JAR); - final HadoopArchives har = new HadoopArchives(mapred.createJobConf()); - assertEquals(0, ToolRunner.run(har, args)); + final String[] args = { "-archiveName", harName, "-p", "input", "*", + "archive" }; + System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH, + HADOOP_ARCHIVES_JAR); + final HadoopArchives har = new HadoopArchives(conf); + Assert.assertEquals(0, ToolRunner.run(har, args)); - //compare results + // compare results final List harPaths = lsr(shell, prefix + harName); - assertEquals(originalPaths, harPaths); + Assert.assertEquals(originalPaths, harPaths); } } - + +@Test public void testPathWithSpaces() throws Exception { fs.delete(archivePath, true); - //create files/directories with spaces + // create files/directories with spaces createFile(inputPath, "c c", fs); final Path sub1 = new Path(inputPath, "sub 1"); fs.mkdirs(sub1); @@ -154,42 +158,36 @@ public void testPathWithSpaces() throws Exception { final Path sub2 = new Path(inputPath, "sub 1 with suffix"); fs.mkdirs(sub2); createFile(sub2, "z", fs); - final Configuration conf = mapred.createJobConf(); + final FsShell shell = new FsShell(conf); final String inputPathStr = inputPath.toUri().getPath(); - System.out.println("inputPathStr = " + inputPathStr); final List originalPaths = lsr(shell, inputPathStr); final URI uri = fs.getUri(); - final String prefix = "har://hdfs-" + uri.getHost() +":" + uri.getPort() + final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort() + archivePath.toUri().getPath() + Path.SEPARATOR; - {//Enable space replacement + {// Enable space replacement final String harName = "foo.har"; - final String[] args = { - "-archiveName", - harName, - "-p", - inputPathStr, - "*", - archivePath.toString() - }; - System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH, HADOOP_ARCHIVES_JAR); - final HadoopArchives har = new HadoopArchives(mapred.createJobConf()); - assertEquals(0, ToolRunner.run(har, args)); + final String[] args = { "-archiveName", harName, "-p", inputPathStr, "*", + archivePath.toString() }; + System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH, + HADOOP_ARCHIVES_JAR); + final HadoopArchives har = new HadoopArchives(conf); + Assert.assertEquals(0, ToolRunner.run(har, args)); - //compare results + // compare results final List harPaths = lsr(shell, prefix + harName); - assertEquals(originalPaths, harPaths); + Assert.assertEquals(originalPaths, harPaths); } } - private static List lsr(final FsShell shell, String dir - ) throws Exception { + private static List lsr(final FsShell shell, String dir) + throws Exception { System.out.println("lsr root=" + dir); - final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); final PrintStream out = new PrintStream(bytes); final PrintStream oldOut = System.out; final PrintStream oldErr = System.err; @@ -197,7 +195,7 @@ private static List lsr(final FsShell shell, String dir System.setErr(out); final String results; try { - assertEquals(0, shell.run(new String[]{"-lsr", dir})); + Assert.assertEquals(0, shell.run(new String[] { "-lsr", dir })); results = bytes.toString(); } finally { IOUtils.closeStream(out); @@ -206,13 +204,13 @@ private static List lsr(final FsShell shell, String dir } System.out.println("lsr results:\n" + results); String dirname = dir; - if (dir.lastIndexOf(Path.SEPARATOR) != -1 ) { + if (dir.lastIndexOf(Path.SEPARATOR) != -1) { dirname = dir.substring(dir.lastIndexOf(Path.SEPARATOR)); } final List paths = new ArrayList(); - for(StringTokenizer t = new StringTokenizer(results, "\n"); - t.hasMoreTokens(); ) { + for (StringTokenizer t = new StringTokenizer(results, "\n"); t + .hasMoreTokens();) { final String s = t.nextToken(); final int i = s.indexOf(dirname); if (i >= 0) { @@ -220,7 +218,8 @@ private static List lsr(final FsShell shell, String dir } } Collections.sort(paths); - System.out.println("lsr paths = " + paths.toString().replace(", ", ",\n ")); + System.out + .println("lsr paths = " + paths.toString().replace(", ", ",\n ")); return paths; } } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java new file mode 100644 index 0000000000..8d6e28ba64 --- /dev/null +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.tools; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.Cluster; +import org.apache.hadoop.mapreduce.JobSubmissionFiles; +import org.apache.hadoop.tools.util.TestDistCpUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.security.Permission; + +public class TestExternalCall { + + private static final Log LOG = LogFactory.getLog(TestExternalCall.class); + + private static FileSystem fs; + + private static String root; + + private static Configuration getConf() { + Configuration conf = new Configuration(); + conf.set("fs.default.name", "file:///"); + conf.set("mapred.job.tracker", "local"); + return conf; + } + + @Before + public void setup() { + + securityManager = System.getSecurityManager(); + System.setSecurityManager(new NoExitSecurityManager()); + try { + fs = FileSystem.get(getConf()); + root = new Path("target/tmp").makeQualified(fs.getUri(), + fs.getWorkingDirectory()).toString(); + TestDistCpUtils.delete(fs, root); + } catch (IOException e) { + LOG.error("Exception encountered ", e); + } + } + + @After + public void tearDown() { + System.setSecurityManager(securityManager); + } +/** + * test methods run end execute of DistCp class. silple copy file + * @throws Exception + */ + @Test + public void testCleanup() throws Exception { + + Configuration conf = getConf(); + + Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), + conf); + stagingDir.getFileSystem(conf).mkdirs(stagingDir); + Path soure = createFile("tmp.txt"); + Path target = createFile("target.txt"); + + DistCp distcp = new DistCp(conf, null); + String[] arg = { soure.toString(), target.toString() }; + + distcp.run(arg); + Assert.assertTrue(fs.exists(target)); + + + } + + private Path createFile(String fname) throws IOException { + Path result = new Path(root + "/" + fname); + OutputStream out = fs.create(result); + try { + out.write((root + "/" + fname).getBytes()); + out.write("\n".getBytes()); + } finally { + out.close(); + } + return result; + } + + /** + * test main method of DistCp. Method should to call System.exit(). + * + */ + @Test + public void testCleanupTestViaToolRunner() throws IOException, InterruptedException { + + Configuration conf = getConf(); + + Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), conf); + stagingDir.getFileSystem(conf).mkdirs(stagingDir); + + Path soure = createFile("tmp.txt"); + Path target = createFile("target.txt"); + try { + + String[] arg = {target.toString(),soure.toString()}; + DistCp.main(arg); + Assert.fail(); + + } catch (ExitException t) { + Assert.assertTrue(fs.exists(target)); + Assert.assertEquals(t.status, 0); + Assert.assertEquals( + stagingDir.getFileSystem(conf).listStatus(stagingDir).length, 0); + } + + } + + private SecurityManager securityManager; + + protected static class ExitException extends SecurityException { + private static final long serialVersionUID = -1982617086752946683L; + public final int status; + + public ExitException(int status) { + super("There is no escape!"); + this.status = status; + } + } + + private static class NoExitSecurityManager extends SecurityManager { + @Override + public void checkPermission(Permission perm) { + // allow anything. + } + + @Override + public void checkPermission(Permission perm, Object context) { + // allow anything. + } + + @Override + public void checkExit(int status) { + super.checkExit(status); + throw new ExitException(status); + } + } +} diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java index 6e4bdf5554..c2259e0eae 100644 --- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java +++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java @@ -43,21 +43,19 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster.Builder; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.tools.DistCpV1; import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Level; -import org.junit.Ignore; /** * A JUnit test for copying files recursively. */ -@Ignore + public class TestCopyFiles extends TestCase { { ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange") @@ -738,20 +736,22 @@ public void testPreserveOption() throws Exception { public void testMapCount() throws Exception { String namenode = null; MiniDFSCluster dfs = null; - MiniMRCluster mr = null; + MiniDFSCluster mr = null; try { Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 3, true, null); + + dfs= new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build(); + FileSystem fs = dfs.getFileSystem(); final FsShell shell = new FsShell(conf); namenode = fs.getUri().toString(); - mr = new MiniMRCluster(3, namenode, 1); MyFile[] files = createFiles(fs.getUri(), "/srcdat"); long totsize = 0; for (MyFile f : files) { totsize += f.getSize(); } - Configuration job = mr.createJobConf(); + + Configuration job = new JobConf(conf); job.setLong("distcp.bytes.per.map", totsize / 3); ToolRunner.run(new DistCpV1(job), new String[] {"-m", "100", @@ -766,8 +766,7 @@ public void testMapCount() throws Exception { System.out.println(execCmd(shell, "-lsr", logdir)); FileStatus[] logs = fs.listStatus(new Path(logdir)); // rare case where splits are exact, logs.length can be 4 - assertTrue("Unexpected map count, logs.length=" + logs.length, - logs.length == 5 || logs.length == 4); + assertTrue( logs.length == 2); deldir(fs, "/destdat"); deldir(fs, "/logs"); diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java index 34435f4d83..53f20b2aec 100644 --- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java +++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java @@ -22,8 +22,6 @@ import java.io.IOException; import java.io.PrintStream; import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; import java.util.Random; import org.apache.commons.logging.LogFactory; @@ -39,10 +37,10 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hadoop.mapred.MiniMRClientClusterFactory; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.log4j.Level; -import org.junit.Ignore; -@Ignore + public class TestDistCh extends junit.framework.TestCase { { ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange") @@ -52,7 +50,8 @@ public class TestDistCh extends junit.framework.TestCase { } static final Long RANDOM_NUMBER_GENERATOR_SEED = null; - + static final FsPermission UMASK = FsPermission.createImmutable((short)0111); + private static final Random RANDOM = new Random(); static { final long seed = RANDOM_NUMBER_GENERATOR_SEED == null? @@ -65,7 +64,7 @@ public class TestDistCh extends junit.framework.TestCase { new Path(System.getProperty("test.build.data","/tmp") ).toString().replace(' ', '+'); - static final int NUN_SUBS = 5; + static final int NUN_SUBS = 7; static class FileTree { private final FileSystem fs; @@ -127,9 +126,12 @@ static class ChPermissionStatus extends PermissionStatus { public void testDistCh() throws Exception { final Configuration conf = new Configuration(); - final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); + + conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+"."+CapacitySchedulerConfiguration.QUEUES, "default"); + conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+".default."+CapacitySchedulerConfiguration.CAPACITY, "100"); + final MiniDFSCluster cluster= new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build(); + final FileSystem fs = cluster.getFileSystem(); - final MiniMRCluster mr = new MiniMRCluster(2, fs.getUri().toString(), 1); final FsShell shell = new FsShell(conf); try { @@ -138,37 +140,36 @@ public void testDistCh() throws Exception { runLsr(shell, tree.root, 0); - //generate random arguments - final String[] args = new String[RANDOM.nextInt(NUN_SUBS-1) + 1]; + final String[] args = new String[NUN_SUBS]; final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS]; - final List indices = new LinkedList(); - for(int i = 0; i < NUN_SUBS; i++) { - indices.add(i); - } - for(int i = 0; i < args.length; i++) { - final int index = indices.remove(RANDOM.nextInt(indices.size())); - final String sub = "sub" + index; - final boolean changeOwner = RANDOM.nextBoolean(); - final boolean changeGroup = RANDOM.nextBoolean(); - final boolean changeMode = !changeOwner && !changeGroup? true: RANDOM.nextBoolean(); - - final String owner = changeOwner? sub: ""; - final String group = changeGroup? sub: ""; - final String permission = changeMode? RANDOM.nextInt(8) + "" + RANDOM.nextInt(8) + "" + RANDOM.nextInt(8): ""; - args[i] = tree.root + "/" + sub + ":" + owner + ":" + group + ":" + permission; - newstatus[index] = new ChPermissionStatus(rootstatus, owner, group, permission); - } - for(int i = 0; i < NUN_SUBS; i++) { - if (newstatus[i] == null) { - newstatus[i] = new ChPermissionStatus(rootstatus); - } - } + + args[0]="/test/testDistCh/sub0:sub1::"; + newstatus[0] = new ChPermissionStatus(rootstatus, "sub1", "", ""); + + args[1]="/test/testDistCh/sub1::sub2:"; + newstatus[1] = new ChPermissionStatus(rootstatus, "", "sub2", ""); + + args[2]="/test/testDistCh/sub2:::437"; + newstatus[2] = new ChPermissionStatus(rootstatus, "", "", "437"); + + args[3]="/test/testDistCh/sub3:sub1:sub2:447"; + newstatus[3] = new ChPermissionStatus(rootstatus, "sub1", "sub2", "447"); + + args[4]="/test/testDistCh/sub4::sub5:437"; + newstatus[4] = new ChPermissionStatus(rootstatus, "", "sub5", "437"); + + args[5]="/test/testDistCh/sub5:sub1:sub5:"; + newstatus[5] = new ChPermissionStatus(rootstatus, "sub1", "sub5", ""); + + args[6]="/test/testDistCh/sub6:sub3::437"; + newstatus[6] = new ChPermissionStatus(rootstatus, "sub3", "", "437"); + System.out.println("args=" + Arrays.asList(args).toString().replace(",", ",\n ")); System.out.println("newstatus=" + Arrays.asList(newstatus).toString().replace(",", ",\n ")); //run DistCh - new DistCh(mr.createJobConf()).run(args); + new DistCh(MiniMRClientClusterFactory.create(this.getClass(), 2, conf).getConfig()).run(args); runLsr(shell, tree.root, 0); //check results @@ -184,7 +185,7 @@ public void testDistCh() throws Exception { } } - static final FsPermission UMASK = FsPermission.createImmutable((short)0111); + static void checkFileStatus(PermissionStatus expected, FileStatus actual) { assertEquals(expected.getUserName(), actual.getOwner()); diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java new file mode 100644 index 0000000000..2042604e77 --- /dev/null +++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.tools; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.EnumSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Test; + +public class TestLogalyzer { + private static String EL = System.getProperty("line.separator"); + private static String TAB = "\t"; + private static final Log LOG = LogFactory.getLog(TestLogalyzer.class); + + private static File workSpace = new File("target", + TestLogalyzer.class.getName() + "-workSpace"); + private static File outdir = new File(workSpace.getAbsoluteFile() + + File.separator + "out"); + + @Test + public void testLogalyzer() throws Exception { + Path f = createLogFile(); + + String[] args = new String[10]; + + args[0] = "-archiveDir"; + args[1] = f.toString(); + args[2] = "-grep"; + args[3] = "44"; + args[4] = "-sort"; + args[5] = "0"; + args[6] = "-analysis"; + args[7] = outdir.getAbsolutePath(); + args[8] = "-separator"; + args[9] = " "; + + Logalyzer.main(args); + checkResult(); + + } + + private void checkResult() throws Exception { + File result = new File(outdir.getAbsolutePath() + File.separator + + "part-00000"); + File success = new File(outdir.getAbsolutePath() + File.separator + + "_SUCCESS"); + Assert.assertTrue(success.exists()); + + FileInputStream fis = new FileInputStream(result); + BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); + String line = br.readLine(); + Assert.assertTrue(("1 44" + TAB + "2").equals(line)); + line = br.readLine(); + + Assert.assertTrue(("3 44" + TAB + "1").equals(line)); + line = br.readLine(); + + Assert.assertTrue(("4 44" + TAB + "1").equals(line)); + + br.close(); + + } + + /** + * Create simple log file + * + * @return + * @throws IOException + */ + + private Path createLogFile() throws IOException { + + FileContext files = FileContext.getLocalFSFileContext(); + + Path ws = new Path(workSpace.getAbsoluteFile().getAbsolutePath()); + + files.delete(ws, true); + Path workSpacePath = new Path(workSpace.getAbsolutePath(), "log"); + files.mkdir(workSpacePath, null, true); + + LOG.info("create logfile.log"); + Path logfile1 = new Path(workSpacePath, "logfile.log"); + + FSDataOutputStream os = files.create(logfile1, + EnumSet.of(CreateFlag.CREATE)); + os.writeBytes("4 3" + EL + "1 3" + EL + "4 44" + EL); + os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL); + os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL); + + os.flush(); + os.close(); + LOG.info("create logfile1.log"); + + Path logfile2 = new Path(workSpacePath, "logfile1.log"); + + os = files.create(logfile2, EnumSet.of(CreateFlag.CREATE)); + os.writeBytes("4 3" + EL + "1 3" + EL + "3 44" + EL); + os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL); + os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL); + + os.flush(); + os.close(); + + return workSpacePath; + } +} diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c9652d8464..3829164627 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -180,6 +180,9 @@ Release 0.23.6 - UNRELEASED YARN-151. Browser thinks RM main page JS is taking too long (Ravi Prakash via bobby) + YARN-204. test coverage for org.apache.hadoop.tools (Aleksey Gorshkov via + bobby) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES