big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-9656. Gridmix unit tests fail on Windows and Linux. Contributed by Chuan Liu.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1496597 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-06-25 19:20:15 +00:00
parent d74908bdb6
commit 8df9a8e6d2
5 changed files with 19 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -559,6 +559,9 @@ Release 2.1.0-beta - UNRELEASED
HADOOP-9439. JniBasedUnixGroupsMapping: fix some crash bugs (Colin Patrick
McCabe)
HADOOP-9656. Gridmix unit tests fail on Windows and Linux. (Chuan Liu via
cnauroth)
BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
HADOOP-8924. Hadoop Common creating package-info.java must not depend on

18
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
View File

@ -86,6 +86,15 @@ class GenerateDistCacheData extends GridmixJob {
"gridmix.distcache.file.list";
static final String JOB_NAME = "GRIDMIX_GENERATE_DISTCACHE_DATA";
/**
* Create distributed cache file with the permissions 0644.
* Since the private distributed cache directory doesn't have execute
* permission for others, it is OK to set read permission for others for
* the files under that directory and still they will become 'private'
* distributed cache files on the simulated cluster.
*/
static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
public GenerateDistCacheData(Configuration conf) throws IOException {
super(conf, 0L, JOB_NAME);
}
@ -146,15 +155,8 @@ public void map(LongWritable key, BytesWritable value, Context context)
String fileName = new String(value.getBytes(), 0, value.getLength());
Path path = new Path(fileName);
/**
* Create distributed cache file with the permissions 0755.
* Since the private distributed cache directory doesn't have execute
* permission for others, it is OK to set read permission for others for
* the files under that directory and still they will become 'private'
* distributed cache files on the simulated cluster.
*/
FSDataOutputStream dos =
FileSystem.create(fs, path, new FsPermission((short)0755));
FileSystem.create(fs, path, new FsPermission(GRIDMIX_DISTCACHE_FILE_PERM));
int size = 0;
for (long bytes = key.get(); bytes > 0; bytes -= size) {

4
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java
View File

@ -311,7 +311,7 @@ static void configureHighRamProperties(Configuration sourceConf,
// set the memory per map task
scaleConfigParameter(sourceConf, destConf,
MRConfig.MAPMEMORY_MB, MRJobConfig.MAP_MEMORY_MB,
JobConf.DISABLED_MEMORY_LIMIT);
MRJobConfig.DEFAULT_MAP_MEMORY_MB);
// validate and fail early
validateTaskMemoryLimits(destConf, MRJobConfig.MAP_MEMORY_MB,
@ -320,7 +320,7 @@ static void configureHighRamProperties(Configuration sourceConf,
// set the memory per reduce task
scaleConfigParameter(sourceConf, destConf,
MRConfig.REDUCEMEMORY_MB, MRJobConfig.REDUCE_MEMORY_MB,
JobConf.DISABLED_MEMORY_LIMIT);
MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
// validate and fail early
validateTaskMemoryLimits(destConf, MRJobConfig.REDUCE_MEMORY_MB,
JTConfig.JT_MAX_REDUCEMEMORY_MB);

3
hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
View File

@ -327,7 +327,8 @@ protected void doSubmission(String jobCreatorName, boolean defaultOutputPath)
final Path out = GridmixTestUtils.DEST.makeQualified(
GridmixTestUtils.dfs.getUri(),
GridmixTestUtils.dfs.getWorkingDirectory());
final Path root = new Path(workspace.getAbsolutePath());
final Path root = new Path(workspace.getName()).makeQualified(
GridmixTestUtils.dfs.getUri(), GridmixTestUtils.dfs.getWorkingDirectory());
if (!workspace.exists()) {
assertTrue(workspace.mkdirs());
}

4
hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
View File

@ -134,8 +134,8 @@ private void validateDistCacheFiles(List<Long> filesSizesExpected, Path distCach
FsPermission perm = stat.getPermission();
assertEquals("Wrong permissions for distributed cache file "
+ stat.getPath().toUri().getPath(), new FsPermission((short) 0644),
perm);
+ stat.getPath().toUri().getPath(), new FsPermission(
GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_PERM), perm);
}
}